class ConcurrentQueue(object):
def __init__(self):
self.en = Lock()
self.de = Lock()
self.queue = deque()
self.de.acquire()
def enqueue(self, ele):
self.en.acquire()
self.queue.append(ele)
self.en.release()
if self.de.locked():
self.de.release()
print(f'enqueue size {len(self.queue)}, {threading.get_ident()}')
def dequeue(self):
self.de.acquire()
val = self.queue.popleft()
if len(self.queue):
self.de.release()
if self.en.locked():
self.en.release()
print(f'dequeue size {len(self.queue)}, {threading.get_ident()}')
return val
class BoundedBlockingQueue(object):
def __init__(self, capacity: int):
self.que = []
self.producer = Lock()
self.customer = Lock()
self.customer.acquire()
self.mutex = Lock()
self.n = capacity
def enqueue(self, element: int) -> None:
self.producer.acquire()
self.que.append(element)
if self.size() != self.n:
self.producer.release()
if self.customer.locked():
self.customer.release()
def dequeue(self) -> int:
self.customer.acquire()
ret = self.que.pop(0)
if self.size():
self.customer.release()
if self.producer.locked():
self.producer.release()
return ret
def _size(self):
return len(self.que)
def size(self) -> int:
ret = None
with self.mutex:
ret = self._size()
return ret
class BoundedBlockingQueue(object):
def __init__(self, capacity: int):
self.enque_lock = Lock()
self.deque_lock = Lock()
self.cap = capacity
self.q = SimpleQueue()
self.deque_lock.acquire()
def enqueue(self, element: int) -> None:
self.enque_lock.acquire()
self.q.put(element)
if self.q.qsize() < self.cap:
self.enque_lock.release()
if self.deque_lock.locked():
self.deque_lock.release()
def dequeue(self) -> int:
self.deque_lock.acquire()
val = None
if self.q.qsize() > 0:
val = self.q.get()
if self.q.qsize():
self.deque_lock.release()
if val and self.enque_lock.locked():
self.enque_lock.release()
return val
def size(self) -> int:
return self.q.qsize()
class BoundedBlockingQueue(object):
def __init__(self, capacity: int):
self.en = Lock()
self.de = Lock()
self.q = deque()
self.capacity = capacity
self.de.acquire()
def enqueue(self, element: int) -> None:
self.en.acquire()
self.q.append(element)
if len(self.q) < self.capacity:
self.en.release()
if self.de.locked():
self.de.release()
def dequeue(self) -> int:
self.de.acquire()
val = self.q.popleft()
if len(self.q):
self.de.release()
if val and self.en.locked():
self.en.release()
return val
def size(self) -> int:
return len(self.q)
class TrafficLight:
def __init__(self):
self.directions12 = (1, 2)
self.dir12_lock = Lock()
self.dir34_lock = Lock()
self.dir12_lock.acquire()
def carArrived(
self,
carId: int, # ID of the car
roadId:
int, # ID of the road the car travels on. Can be 1 (road A) or 2 (road B)
direction: int, # Direction of the car
turnGreen:
'Callable[[], None]', # Use turnGreen() to turn light to green on current road
crossCar:
'Callable[[], None]' # Use crossCar() to make car cross the intersection
) -> None:
if direction in self.directions12:
if self.dir34_lock.locked(): # Other roda is green
self.dir34_lock.release()
self.dir12_lock.acquire()
turnGreen()
else:
if self.dir12_lock.locked():
self.dir12_lock.release()
self.dir34_lock.acquire()
turnGreen()
crossCar()
class KeyEventQueue:
def __init__(self):
self.__lock = Lock()
self.__queue = []
def append(self, kevt: KeyEvent):
try:
self.__lock.acquire()
self.__queue.append(kevt)
except:
raise
finally:
if self.__lock.locked():
self.__lock.release()
def take(self) -> KeyEvent:
try:
self.__lock.acquire()
v = self.__queue.pop(0) if self.__queue else None
except:
raise
finally:
if self.__lock.locked():
self.__lock.release()
return v
def clear(self):
self.__queue.clear()
def useLock():
lock = Lock()
with lock:
print(lock.locked())
print("lock is available")
# print(lock.acquire(timeout=3))
print(lock.locked())
class Condition(object):
"Pipe-based condition uses poll to mimic build-in Condition"
MSECSMAX = 0x7FFFFFFF
def __init__ (self):
self._lock = Lock()
self._lockset = set()
self._poll = select.poll()
self._readfd, self._writefd = os.pipe()
self._poll.register(self._readfd, select.POLLIN)
super(Condition, self).__init__()
def acquire(self):
return self._lock.acquire()
def release(self):
return self._lock.release()
def locked(self):
return self._lock.locked()
def wait(self, timeout=None):
assert self._lock.locked(), 'wait called on un-acquire()d lock'
waitlock = allocate_lock()
waitlock.acquire()
self._lockset.add(waitlock)
self._lock.release()
try:
if timeout is None:
self._waituntil(waitlock)
else:
self._waitupto(timeout, waitlock)
finally:
self._lock.acquire()
if waitlock.acquire(0):
os.read(self._readfd, 1)
self._lockset.remove(waitlock)
def _waitupto(self, timeout, waitlock):
startuptime = uptime.secs()
while not timeout < 0:
msecs = min(timeout * 1000, self.MSECSMAX)
readable = self._pollreadable(msecs)
if waitlock.locked():
curuptime = uptime.secs()
lapsetime = curuptime - startuptime
timeout = timeout - lapsetime
else:
break
else:
return False
return True
def _waituntil(self, waitlock):
while waitlock.locked():
self._pollreadable()
def _pollreadable(self, timeout = None):
try:
return self._poll.poll(timeout)
except select.error, error:
msglog.log('broadway', msglog.types.WARN,
'Notifier ignoring %s' % (error,))
return False
def _lock():
# 锁
ll = Lock()
ll.acquire()
ll.release()
ll.locked()
rl = RLock()
rl.acquire()
rl.release()
class StandardServo:
max_degree = 180
min_freq = 1100
max_freq = 1900
def __init__(self, pin):
self.control = None
self.pin = pin
self._motor_initialize()
self.running = False
self.angle = None
self.write_lock = Lock()
self.write_thread = None
self.start()
def _motor_initialize(self):
self.control = kit.servo[self.pin]
# self.control.actuation_range = self.max_degree
# self.control.set_pulse_width_range(self.min_freq, self.max_freq)
# self.change_angle(0)
def start(self):
if self.running:
print('Motor is already running')
return None
else:
self.running = True
self.angle = 0
self.write_thread = Thread(target=self._motor_thread)
self.write_thread.start()
return self
def _motor_thread(self):
while self.running:
self.write_lock.acquire()
self.control.angle = self.angle
def change_angle(self, angle):
if angle != self.angle:
self.angle = angle
if self.write_lock.locked():
self.write_lock.release()
def stop(self):
print(self.pin, "motor is shutting down...")
if self.running:
self.running = False
if self.write_lock.locked():
self.write_lock.release()
self.write_thread.join()
print(self.pin, "motor is shut down...")
else:
print(self.pin, "motor is already shut down...")
class market_thread(market, Thread):
def __init__(self, sync = True,data_path = '', animation_speed = 1.0, random=False,length=0, graph=False, graph_span=50, cos_mrkt_data = None, type = None):
'''/*
* sync: True: wait for all agent finish to step into next unit of time
* number_of_agent : in case you want more than one agent
* animation_speed: if not sync, the market will move forward animation_speed unit per second
* eg: if animation_speed = 0.5 , then market will move forward 1 unit of time every 2 seconds.
* graph: enable real time market grpah
* graph_span: How many past units of time are included in the graph. 0 : Entire history
* cos_mrkt_data: import customized mrkt_data
*/'''
if(cos_mrkt_data is not None):
mrkt_data.__init__ = cos_mrkt_data
print("market type is:", type)
market.__init__(self, random=random, length=length,data_path = data_path, type = type)
Thread.__init__(self)
self.sync = sync
#self.next_time_status = False
self.next_time_mutex = Lock()
self.exit = False
self.animation_speed = animation_speed
def set_next_time_status(self, value=True):
self.next_time_mutex.acquire()
while (self.next_time_mutex.locked()):
pass
#self.next_time_status = value
def set_exit(self, value=True):
# used to end the thread
self.exit = value
def run(self) -> None:
while True:
if self.ended: # if the simulation runs out
break
if self.exit: # be terminated
break
if(self.sync):
if(self.next_time_mutex.locked()):
self.next_time()
self.next_time_mutex.release()
else:
time.sleep(1/self.animation_speed)
self.next_time()
class LockableClass(object):
def __init__(self):
self._lock = Lock()
def test_method(self):
lock_state = self._lock.locked() # pylint: disable=no-member
return lock_state
@synchronized
def sync_test_method(self):
lock_state = self._lock.locked() # pylint: disable=no-member
return lock_state
class LockableClass(object):
def __init__(self):
self._lock = Lock()
def test_method(self):
lock_state = self._lock.locked() # pylint: disable=no-member
return lock_state
@synchronized
def sync_test_method(self):
lock_state = self._lock.locked() # pylint: disable=no-member
return lock_state
class SuccessLock:
def __init__(self):
self.lock = Lock()
self.pause = Lock()
self.code = 0L
self.success = False
self.finished = True
def reset(self):
self.success = False
self.finished = False
def set(self):
self.lock.acquire()
if not self.pause.locked():
self.pause.acquire()
self.first = True
self.code += 1L
self.lock.release()
return self.code
def trip(self, code, s=False):
self.lock.acquire()
try:
if code == self.code and not self.finished:
r = self.first
self.first = False
if s:
self.finished = True
self.success = True
return r
finally:
self.lock.release()
def give_up(self):
self.lock.acquire()
self.success = False
self.finished = True
self.lock.release()
def wait(self):
self.pause.acquire()
def unwait(self, code):
if code == self.code and self.pause.locked():
self.pause.release()
def isfinished(self):
self.lock.acquire()
x = self.finished
self.lock.release()
return x
class SuccessLock:
def __init__(self):
self.lock = Lock()
self.pause = Lock()
self.code = 0L
self.success = False
self.finished = True
def reset(self):
self.success = False
self.finished = False
def set(self):
self.lock.acquire()
if not self.pause.locked():
self.pause.acquire()
self.first = True
self.code += 1L
self.lock.release()
return self.code
def trip(self, code, s = False):
self.lock.acquire()
try:
if code == self.code and not self.finished:
r = self.first
self.first = False
if s:
self.finished = True
self.success = True
return r
finally:
self.lock.release()
def give_up(self):
self.lock.acquire()
self.success = False
self.finished = True
self.lock.release()
def wait(self):
self.pause.acquire()
def unwait(self, code):
if code == self.code and self.pause.locked():
self.pause.release()
def isfinished(self):
self.lock.acquire()
x = self.finished
self.lock.release()
return x
class MonitoringThread(Thread, Button):
def __init__(self, pin: int):
Thread.__init__(self)
Button.__init__(self, pin, pull_up=None, active_state=False)
self.when_activated = self.contact
self.when_deactivated = self.released
self.lock_active = Lock()
self.lock_inactive = Lock()
self.lock_active.acquire()
self.exit_event = Event()
def contact(self):
self.lock_active.release()
self.lock_inactive.acquire()
def released(self):
self.lock_inactive.release()
self.lock_active.acquire()
def exit(self):
self.exit_event.set()
if self.lock_active.locked():
self.lock_active.release()
if self.lock_inactive.locked():
self.lock_inactive.release()
def print_part(self, s: str):
sys.stdout.write(s)
sys.stdout.flush()
def run(self):
print(
f'Starting impulse counter at {self.pin} at {datetime.now():%Y-%m-%d %H:%M:%S}'
)
contacts = 0
while not self.exit_event.is_set():
self.print_part('waiting...')
self.lock_active.acquire()
if self.exit_event.is_set():
break
contacts += 1
mark = datetime.now()
self.print_part(
f'{BCK*10}[{contacts:03d}] {mark:%Y-%m-%d %H:%M:%S} ')
self.lock_active.release()
self.lock_inactive.acquire()
print(f'{(datetime.now() - mark).total_seconds():3.4} seconds')
self.lock_inactive.release()
class Store:
def __init__(self):
self.items = {}
self.__lock__ = Lock()
def lock(self):
if self.locked():
return
self.__lock__.acquire()
def release(self):
if not self.locked():
return
self.__lock__.release()
def locked(self):
return self.__lock__.locked()
def add(self, instance):
from resources.models import Resource
self.items.update({
instance.id : RespaOutlookManager(instance)
})
res = Resource.objects.get(pk=instance.resource.id)
if res:
res.configuration = instance
res.save()
class GenericMindNode:
def __init__(self, name="GenericMindNode", node_type=node_types.GENERIC_NODE):
self.name = name
self.node_type = node_type
self.client_CommandConcept = None
self.subscriber_PerceptionConcept = None
self.sensor_cb_lock = Lock()
self.sensor_cb_thread = None
def sensor_callback(self, perception_concept: msg.PerceptionConcept):
rospy.loginfo("From SensorInterpreter: %s:%s" % (perception_concept.symbol, perception_concept.modifier))
def _sensor_callback_thread(self, perception_concept: msg.PerceptionConcept):
rospy.loginfo("sensor_callback thread")
if self.sensor_cb_lock.acquire(blocking=False):
self.sensor_callback(perception_concept)
self.sensor_cb_lock.release()
def _sensor_callback(self, perception_concept: msg.PerceptionConcept):
if self.sensor_cb_lock.locked():
rospy.loginfo("sensor_callback is busy!")
else:
self.sensor_cb_thread = Thread(target=self._sensor_callback_thread, args=(perception_concept,), daemon=True)
self.sensor_cb_thread.start()
def emotion_callback(self, params: msg.EmotionParams):
params_dict = json.loads(params.params_json)
rospy.loginfo(f"Params: ${str(params_dict)}")
def _init_communication(self):
rospy.logdebug("Client \'%s\' is starting..." % self.name)
self.client_CommandConcept = rospy.ServiceProxy(zros.services.CONCEPT_TO_COMMAND, srv.CommandConcept)
self.subscriber_PerceptionConcept = zros.get.subscriber(topic_name=zros.topics.MAIN_SENSOR_INTERPRETER,
data_class=msg.PerceptionConcept,
callback=self._sensor_callback)
self.subscriber_EmotionParams = zros.get.subscriber(topic_name=zros.topics.EMOTION_PARAMS,
data_class=msg.EmotionParams,
callback=self.emotion_callback)
rospy.loginfo("Client \'%s\' is ready" % self.name)
def start(self):
rospy.logdebug("Node \'%s\' is starting..." % self.name)
rospy.init_node(self.name, anonymous=False)
self._init_communication()
rospy.loginfo("[ DONE ] Node \'%s\' is ready..." % self.name)
def to_will_cb(self, symbol: str, modifier: tuple = ()) -> str:
"""
Sends concept requests from callbacks
"""
rospy.loginfo("The mind is willing: %s:%s" % (symbol, str(modifier)))
try:
resp = self.client_CommandConcept(self.node_type, symbol, str(modifier))
rospy.logwarn("Response for %s: %s" % (symbol, resp))
return resp.result
except rospy.ServiceException as e:
print("Service call failed: %s" % e)
def __call__(self):
self.start()
class BoundedBlockingQueue(object):
def __init__(self, capacity: int):
self.locke, self.lockd = Lock(), Lock()
self.dq = deque()
self.capacity = capacity
self.lockd.acquire() # Lock dequeue first because it's empty
def enqueue(self, element: int) -> None:
self.locke.acquire()
self.dq.append(element)
if len(self.dq) < self.capacity:
self.locke.release()
if self.lockd.locked(
): # if dequeue locked, release it since now we have element
self.lockd.release()
def dequeue(self) -> int:
self.lockd.acquire()
res = self.dq.popleft()
if len(self.dq):
self.lockd.release()
if res and self.locke.locked(
): # if enqueue locked, release it since now we have element
self.locke.release()
return res
def size(self) -> int:
return len(self.dq)
def __init__(self, username, key, throttle_seconds=None, **kwargs):
"""
:param username: Your Adafruit username
:type username: str
:param key: Your Adafruit IO key
:type key: str
:param throttle_seconds: If set, then instead of sending the values directly over ``send`` the plugin will
first collect all the samples within the specified period and then dispatch them to Adafruit IO.
You may want to set it if you have data sources providing a lot of data points and you don't want to hit
the throttling limitations of Adafruit.
:type throttle_seconds: float
"""
global data_throttler_lock
super().__init__(**kwargs)
self._username = username
self._key = key
self.aio = Client(username=username, key=key)
self.throttle_seconds = throttle_seconds
if not data_throttler_lock:
data_throttler_lock = Lock()
if self.throttle_seconds and not data_throttler_lock.locked():
self._get_redis()
self.logger.info('Starting Adafruit IO throttler thread')
data_throttler_lock.acquire(False)
self.data_throttler = Thread(target=self._data_throttler())
self.data_throttler.start()
class ConcurrentPopulations:
def __init__(self):
self._lock = Lock()
self._generator = None
self._discriminator = None
def lock(self):
self._lock.acquire()
def unlock(self):
self._lock.release()
@property
def generator(self):
return self._generator
@generator.setter
def generator(self, value):
self._generator = value
@property
def discriminator(self):
return self._discriminator
@discriminator.setter
def discriminator(self, value):
self._discriminator = value
def locked(self):
return self._lock.locked()
def __init__(self, path: str, lock: Lock, who: str = ""):
self.conn = sqlite3.connect(path)
if who:
self.whoami = who
assert lock.locked()
self.lock = lock
print("start connection for {}".format(self.whoami))
def worker(q: Queue, stop: Lock):
ex = {'client': 'WORKER'}
while not stop.locked():
# For exit thread in stopping simulation.
try:
task_id = q.get(timeout=QUEUE_GET_TIMEOUT)
except Empty:
continue
logger.info('New task - {}'.format(TASK_TYPES[task_id]), extra=ex)
search_time = uniform(*TIME_SEARCH_EDUCATIONAL) \
if task_id == TASK_EDUCATIONAL else uniform(*TIME_SEARCH_TECHNICAL)
sleep(search_time)
logger.info('Task completed in {:.2f} min'.format(search_time),
extra=ex)
output_time = uniform(*TIME_RESULT_OUTPUT)
sleep(output_time)
logger.info('Results output in {:.2f} min'.format(output_time),
extra=ex)
q.task_done()
class FrameBuffer(object):
def initialize(self, width, height, dtype):
self._shape = (height, width)
self._buffers = [
np.zeros(self._shape, dtype=dtype),
np.zeros(self._shape, dtype=dtype)
]
self._write_index = 0
self._read_index = 1
self._lock = Lock()
def write(self, data):
source = data.reshape(self._shape)
destination = self._buffers[self._write_index]
np.copyto(dst=destination, src=source)
self._swap_buffers()
def read(self):
assert self._lock.locked()
return self._buffers[self._read_index]
def read_lock(self):
return self._lock
def _swap_buffers(self):
with self._lock:
self._write_index, self._read_index = (self._read_index,
self._write_index)
class SingleThread():
def __init__(self, progressbar):
self.lock = Lock()
self.progressbar = progressbar
def run_with_progressbar(self, method, args=None, text=None, no_thread=False):
if no_thread:
if method and args:
method(args)
if method:
method()
else:
self.progressbar.start(text)
self._run(method, args)
def _run(self, method, args=None):
if not self.lock.locked():
self.lock.acquire()
thread.start_new_thread(self._thread_task, (method, args,))
else:
logging.warning("Thread not finished" + str(method) + str(args))
def _thread_task(self, method, args):
try:
if method and args:
method(args)
elif method:
method()
time.sleep(0.1)
except Exception, e:
logging.error(method.__name__ + "(" + str(args) + "):" + str(e))
finally:
class Fork():
"""
Fork is a shared resource among Philosophers. A Fork
can only be held by one Philosopher at a time
"""
counter = 0
timeout = 2 # in seconda
def __init__(self):
self.lock = Lock()
self.id = Fork.counter
Fork.counter += 1
def is_used(self):
return self.lock.locked()
def pickup(self):
start = time.time()
got_lock = False
while time.time() - start < self.timeout and not got_lock:
got_lock = self.lock.acquire(0)
if not got_lock:
logging.error('Deadlock')
exit()
def putdown(self):
self.lock.release()
def manage_data():
mutex = Lock()
try:
# Calculate what was unix time 6 months ago
seconds_in_six_months = 180 * 60 * 60 * 24
unix_time_now = time.time()
unix_time_six_months_ago = unix_time_now - seconds_in_six_months
karma_logs_db_conn = sqlite3.connect('karma_logs.db', check_same_thread=False)
karma_logs_db_cursor = karma_logs_db_conn.cursor()
mutex.acquire()
# delete the record of comments that are of older than 6 months
karma_logs_db_cursor.execute(
"DELETE FROM comments WHERE submission_created_utc <= '{}'".format(unix_time_six_months_ago))
# commit
karma_logs_db_conn.commit()
mutex.release()
user_database_obj.archive_data()
user_database_obj.erase_data()
print("Old data deleted " + time.strftime('%I:%M %p %Z'))
except Exception as main_loop_exception:
if mutex.locked():
mutex.release()
tb = traceback.format_exc()
print(main_loop_exception)
try:
send_message_to_discord(tb)
except Exception as discord_exception:
print("Error sending message to discord", str(discord_exception))
class LoginScreenVM:
def __init__(self):
self.__auth = SpotifyAuth.get_instance()
self.success = False
self.lock = Lock()
def login(self, write_func):
if self.lock.locked():
write_func("Please only attempt one login at a time.")
return
self.lock.acquire()
old_stderr = sys.stderr
sys.stderr = open(os.devnull, "w")
if self.success:
write_func("You are already logged in!")
return
try:
if self.__auth.establish_connection():
self.success = True
write_func(f"Welcome, {self.__auth.current_user}!")
else:
self.success = False
write_func("Login cancelled by user")
except Exception:
self.success = False
write_func("An unknown login error occurred.")
finally:
self.lock.release()
sys.stderr = old_stderr
class Answer(list):
"""Safe storage of partial answers from server.
When client try to acquire for reading, it will be locked until full
response form client has been received.
The answer is automatically locked at instantation by the current, so
the only way that a client could consume the data is by releasing by
the server side when answer is full filled.
This is intended for Answers to be created in the network mainloop that
will be process the partial responses from server.
"""
def __init__(self, *args, **kw):
self._lock = Lock()
self.call_args = None
self.error = None
self.return_code = None
super(Answer, self).__init__(*args, **kw)
self.acquire()
def acquire(self, timeout=-1):
"Try to lock the answer to fill it up or use it."
return self._lock.acquire(timeout=timeout)
def release(self):
"Release the lock, usually done by server thread."
if self._lock.locked():
self._lock.release()
else:
print('Warning: trying to release an unlocked mutex')
@property
def values(self):
"Convert the values to a ordinary list"
return list(self)
class Locker:
"""
Allow the connected_objects to be use by only one
thread, but also to be kill if another thread need it
"""
def __init__(self):
self.mutex = Lock()
self.mutex_killer = Lock()
self.killer = False
def lock(self):
if not (self.killer) and self.mutex.locked():
self.kill()
self.mutex.acquire()
self.mutex_killer.acquire()
self.killer = False
self.mutex_killer.release()
def unlock(self):
self.mutex.release()
def test(self):
return self.killer
def kill(self):
self.mutex_killer.acquire()
self.killer = True
self.mutex_killer.release()
def terminal(q: Queue, task_type: int, stop: Lock):
ex = {'client': '{} TERMINAL'.format(TASK_TYPES[task_type])}
while not stop.locked():
# For exit thread in stopping simulation.
try:
task_id = q.get(timeout=QUEUE_GET_TIMEOUT)
except Empty:
continue
logger.debug('New request', extra=ex)
prepare_time = uniform(*TIME_REQUEST_PREPARE)
sleep(prepare_time)
logger.debug('Results prepared in {:.2f} min'.format(prepare_time),
extra=ex)
search_time = uniform(*TIME_SEARCH_EDUCATIONAL) \
if task_id == TASK_EDUCATIONAL else uniform(*TIME_SEARCH_TECHNICAL)
sleep(search_time)
logger.debug('Task completed in {:.2f} min'.format(search_time),
extra=ex)
output_time = uniform(*TIME_RESULT_OUTPUT)
sleep(output_time)
logger.debug('Results output in {:.2f} min'.format(output_time),
extra=ex)
q.task_done()
class BlaLock(object):
"""
Simple wrapper class for the thread.lock class which only raises an
exception when trying to release an unlocked lock if it's initialized with
strict=True.
"""
def __init__(self, strict=False, blocking=True):
self.__strict = strict
self.__blocking = blocking
self.__lock = Lock()
def acquire(self):
self.__lock.acquire(self.__blocking)
def release(self):
try:
self.__lock.release()
except ThreadError:
if self.__strict:
raise
def locked(self):
return self.__lock.locked()
def __enter__(self, *args):
self.acquire()
def __exit__(self, *args):
self.release()
class TaskQueue(Thread):
def __init__(self,
batch_processor: BatchProcessor,
queue_capacity: int = 0,
max_wait_time: float = 0.5):
super().__init__()
self._interval = max_wait_time
self._batch_processor = batch_processor
self._stop = False
self._queue = Queue(maxsize=queue_capacity)
self._execution_lock = Lock()
self._count_trigger = Lock()
def _size_overflow(self):
return self._queue.qsize() >= self._batch_processor.get_batch_size()
def stop(self):
self._stop = True
def async_submit(self, data) -> Future:
if not self._stop:
data_pack = DataPack(data)
self._queue.put(data_pack)
locked = self._execution_lock.acquire(blocking=False)
if locked:
if self._size_overflow() and self._count_trigger.locked():
self._count_trigger.release()
self._execution_lock.release()
# else means failed to get lock, batch processor is running
return data_pack
else:
raise Exception("Task queue was stopped to accept task.")
def _process(self):
log.debug("[BP] Process starting, waiting for lock")
with self._execution_lock:
log.debug("[BP] Lock acquired")
self._batch_processor.process([
self._queue.get() for _ in range(
min(self._queue.qsize(),
self._batch_processor.get_batch_size()))
])
self._count_trigger.acquire(blocking=False)
def run(self) -> None:
log.info("Batch service started.")
while not (self._stop and self._queue.qsize() == 0):
if self._stop:
log.info("Exit flag set, cleaning queue...")
try:
if self._queue.qsize() < self._batch_processor.get_batch_size(
) and not self._stop:
self._count_trigger.acquire(timeout=self._interval)
if self._queue.qsize() > 0:
self._process()
except Exception as ex:
log.fatal("Some critical error happened while doing batch",
exc_info=ex)
time.sleep(1)
log.info("Queue cleaned, exiting batch process.")
class Channel:
""" Channel allows communication/synchronization between threads"""
def __init__(self):
self.message = {}
self.write_mutex = Lock()
self.read_mutex = Lock()
self.read_mutex.acquire()
def write(self, msg):
""" Write allows a message to be sent down the channel"""
self.write_mutex.acquire()
self.message = msg
self.read_mutex.release()
def read(self):
""" read allows a message to be read from the channel """
self.read_mutex.acquire()
msg = self.message
self.message = {}
self.write_mutex.release()
return msg
def is_full(self):
""" return true if we can't write to the channel because it hasn't been read from yet """
return self.write_mutex.locked()
class Terminal:
lock = None
app = None
def __init__(self, app):
self.lock = Lock()
self.lock.acquire()
self.app = app
self.app.debug_print = lambda cmd: eval(cmd)
def show_terminal(self):
if self.lock.locked():
self.lock.release()
def pause_terminal(self):
if not self.lock.locked():
self.lock.acquire()
def run_terminal(self):
with self.lock:
printc("\n\nInput commands to RCAT below. Type help for list for commands.", "blue")
while 1:
self.lock.acquire()
sys.stdout.write(ansi_codes["green"] + "[rcat]: ")
line = sys.stdin.readline()
if line.startswith("quit"):
printc("Quitting RCAT....", "yellow")
sys.exit(0)
if line.startswith("help"):
printc(
"quit: (Force) quits RCAT\nprint arg: Runs a 'print arg' in the application. Application represented by 'app' (e.g. print app)",
"endc",
)
if line.startswith("restart"):
self.app.declare_game_end()
if line.startswith("print"):
try:
cmd = line.split()
if len(cmd) > 2:
logging.warn("Only one variable at a time.")
else:
app = self.app
print eval(cmd[1])
except Exception, e:
logging.error("Could not read variable.")
print e
self.show_terminal()
class Renderer(Thread):
def __init__(self):
Thread.__init__(self)
self.x = 0
self.snapshot = 0
self.param_update = Lock()
print_(self.param_update.locked())
self.param_ready = Lock()
self.param_ready.acquire() # not ready
self.ss_update = Lock() # not wanted
self.ss_ready = Lock()
self.ss_ready.acquire() # not ready
self.param_lock = Lock()
self.print_ = get_print("Rnd", Fore.YELLOW)
def set_param(self, x):
self.x = x
def draw(self):
sleep_time = randint(0,5)/10.
self.print_("drawing")
# if self.param_update.locked():
# # if it is updating, wait until it is ready
# self.param_ready.acquire()
self.param_update.acquire()
self.print_("using param - %d" % self.x)
sleep(sleep_time) # rendering to frame buffer
self.snapshot = self.x # swap frame buffer
self.print_("frame buffer - %d" % self.snapshot)
if self.ss_update.locked():
self.save_snapshot()
self.ss_ready.release()
self.param_update.release()
sleep(0.5 - sleep_time)
def save_snapshot(self):
self.print_("saving snapshot - %d - %d"%(self.snapshot, self.x))
def run(self):
while True:
self.draw()
class Locker(metaclass=Singleton):
def __init__(self):
self.lock = Lock()
def set_acquire(self):
self.lock.acquire()
def set_release(self):
self.lock.release()
def is_locked(self):
return self.lock.locked()
def printID(self):
return id(self.lock)
class SingleThread():
def __init__(self, progressbar=None):
self.lock = Lock()
self.progressbar = progressbar
def run_with_progressbar(self, method, args=None, text='', no_thread=False, with_lock=True):
#with_lock - shows, does it necessarily to do a lock or not
if no_thread:
if method and args:
method(args)
if method:
method()
else:
self._run(method, args, text, with_lock)
def _run(self, method, args=None, text='', with_lock=True):
if not self.lock.locked():
self.lock.acquire()
if self.progressbar:
self.progressbar.start(text)
thread.start_new_thread(self._thread_task, (method, args,))
else:
logging.warning("Previous thread not finished " + str(method) + " " + str(args))
if not with_lock:
logging.info("Try to run method without progress bar")
thread.start_new_thread(self._thread_task, (method, args))
def _thread_task(self, method, args, with_lock=True):
try:
if method and args:
method(args)
elif method:
method()
except Exception, e:
logging.error(str(e))
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback, file=sys.stdout)
finally:
class BaseFoobnixControls():
def __init__(self):
self.vk_service = VKService(FC().access_token, FC().user_id)
self.count_errors = 0
self.is_scrobbled = False
self.start_time = None
self.cache_text = None
self.play_lock = Lock()
def check_for_media(self, args):
dirs = []
files = []
for arg in args:
if os.path.isdir(arg):
dirs.append(arg)
elif os.path.isfile(arg) and get_file_extension(arg) in FC().all_support_formats:
files.append(arg)
if dirs:
self.on_add_folders(dirs)
elif files:
self.on_add_files(files)
try:
self.play_first_added(files)
except:
logging.error("Can't to play first added file")
def play_first_added(self, added_files):
tree = self.notetabs.get_current_tree()
model = tree.get_model()
number = len(model) - len(added_files)
if (number) > -1:
iter = model.get_iter_from_string(str(number))
bean = tree.get_bean_from_model_iter(model, iter)
tree.set_play_icon_to_bean(bean)
self.play(bean)
def love_this_tracks(self, beans=None):
if not beans:
return
map(self.lastfm_service.love, beans)
def add_to_my_playlist(self, beans=None):
if not beans:
return
map(self.vk_service.add, beans)
def copy_link(self, beans=None):
if not beans:
return
if hasattr(beans[0], 'path'):
pyperclip.copy(beans[0].path)
if FC().notifier:
notification = Notify.Notification.new("In clipboard", beans[0].path, "")
notification.set_urgency(Notify.Urgency.LOW)
notification.set_timeout(FC().notify_time)
notification.show()
def show_google_results(self, query):
return [FModel('"%s" not found' % query)]
def get_active_bean(self):
tree = self.notetabs.get_current_tree()
if tree:
return tree.get_selected_or_current_bean()
def play_selected_song(self):
current = self.get_active_bean()
tree = self.notetabs.get_current_tree()
if not current:
try:
current = tree.get_bean_under_pointer_icon()
except AttributeError:
return
if not current:
return None
logging.debug("play current bean is %s" % str(current.text))
if current and current.is_file:
tree.set_play_icon_to_bean(current)
self.play(current)
def check_path(self, path):
if path:
if not path.startswith("http://"):
if os.path.exists(path):
return True
else:
try:
u = urlopen(path, timeout=5) # @UnusedVariable
if "u" not in vars():
return False
return True
except:
return False
return False
def save_beans_to(self, beans):
return None
def on_chage_player_state(self, state, bean):
logging.debug("bean state %s" % state)
self.set_dbus_state(state, bean)
if not FC().system_icons_dinamic:
return None
if state == STATE_STOP:
self.trayicon.set_image_from_path(FC().stop_icon_entry)
elif state == STATE_PAUSE:
self.trayicon.set_image_from_path(FC().pause_icon_entry)
elif state == STATE_PLAY:
self.trayicon.set_image_from_path(FC().play_icon_entry)
if bean and bean.type:
logging.debug("bean state and type %s %s" % (state, bean.type))
if bean.type == FTYPE_RADIO:
return self.trayicon.set_image_from_path(FC().radio_icon_entry)
@idle_task
def set_dbus_state(self, state, bean):
if self.dbus:
self.dbus._update_info(bean)
if state is STATE_PLAY:
self.dbus._set_state_play()
elif state is STATE_PAUSE:
self.dbus._set_state_pause()
else:
self.dbus._set_state_stop()
def on_add_folders(self, paths=None):
if not paths:
paths = directory_chooser_dialog(_("Choose folders to open"), FC().last_dir)
if not paths:
return
tree = self.notetabs.get_current_tree()
FC().last_dir = os.path.dirname(paths[0])
if tree.is_empty():
if len(paths) > 1:
tabname = os.path.basename(FC().last_dir)
else:
tabname = os.path.basename(paths[0])
self.notetabs.rename_tab(tree.scroll, tabname)
tree.append(paths)
def on_add_files(self, paths=None, tab_name=None):
if not paths:
paths = file_chooser_dialog(_("Choose file to open"), FC().last_dir)
if not paths:
return
tree = self.notetabs.get_current_tree()
FC().last_dir = os.path.dirname(paths[0])
if tree.is_empty():
tabname = os.path.split(os.path.dirname(paths[0]))[1]
self.notetabs.rename_tab(tree.scroll, tabname)
tree.append(paths)
def set_playlist_tree(self):
self.notetabs.set_playlist_tree()
def set_playlist_plain(self):
self.notetabs.set_playlist_plain()
def load_music_tree(self):
tabs = len(FCache().cache_music_tree_beans)
tabhelper = self.perspectives.get_perspective('fs').get_tabhelper()
for tab in xrange(tabs - 1, -1, -1):
tabhelper._append_tab(FCache().tab_names[tab], rows=FCache().cache_music_tree_beans[tab])
if not FCache().cache_music_tree_beans[tab]:
self.perspectives.get_perspective('fs').show_add_button()
else:
self.perspectives.get_perspective('fs').hide_add_button()
logging.info("Tree loaded from cache")
if FC().update_tree_on_start:
def cycle():
for n in xrange(len(FCache().music_paths)):
tab_child = tabhelper.get_nth_page(n)
tree = tab_child.get_child()
self.update_music_tree(tree, n)
GLib.idle_add(cycle)
def update_music_tree(self, tree, number_of_page=0):
logging.info("Update music tree" + str(FCache().music_paths[number_of_page]))
tree.clear_tree() # safe method
FCache().cache_music_tree_beans[number_of_page] = {}
all = get_all_music_by_paths(FCache().music_paths[number_of_page], self)
try:
self.perspectives.get_perspective('fs').hide_add_button()
except AttributeError:
logging.warn("Object perspective not exists yet")
if not all:
try:
self.perspectives.get_perspective('fs').show_add_button()
except AttributeError:
logging.warn("Object perspective not exists yet")
tree.append_all(all) # safe method
tree.ext_width = tree.ext_column.get_width()
GLib.idle_add(tree.save_rows_from_tree,
FCache().cache_music_tree_beans[number_of_page])
#GLib.idle_add(self.tabhelper.on_save_tabs) # for true order
@idle_task
def set_visible_video_panel(self, flag):
return
#FC().is_view_video_panel = flag
#if flag:
# self.movie_window.show()
#else:
# self.movie_window.hide()
@idle_task
def volume_up(self):
self.volume.volume_up()
@idle_task
def volume_down(self):
self.volume.volume_down()
@idle_task
def mute(self):
self.volume.mute()
@idle_task
def hide(self):
self.main_window.hide()
@idle_task
def show_hide(self):
self.main_window.show_hide()
@idle_task
def show(self):
self.main_window.show()
@idle_task
def play_pause(self):
if self.media_engine.get_state() == STATE_PLAY:
self.media_engine.state_pause()
elif self.media_engine.get_state() == STATE_STOP:
self.state_play(True)
else:
self.media_engine.state_play()
@idle_task
def seek_up(self):
self.media_engine.seek_up()
@idle_task
def seek_down(self):
self.media_engine.seek_down()
@idle_task
def windows_visibility(self):
visible = self.main_window.get_property('visible')
if visible:
GLib.idle_add(self.main_window.hide)
else:
GLib.idle_add(self.main_window.show)
@idle_task
def state_play(self, under_pointer_icon=False):
if self.media_engine.get_state() == STATE_PAUSE:
self.media_engine.state_play()
self.statusbar.set_text(self.media_engine.bean.info)
elif under_pointer_icon:
tree = self.notetabs.get_current_tree()
bean = tree.get_bean_under_pointer_icon()
self.play(bean)
else:
self.play_selected_song()
@idle_task
def show_preferences(self):
self.preferences.show()
@idle_task
def state_pause(self):
self.media_engine.state_pause()
@idle_task_priority(priority=GLib.PRIORITY_HIGH_IDLE)
def state_stop(self, remember_position=False):
self.record.hide()
self.media_engine.state_stop(remember_position)
if not remember_position:
self.statusbar.set_text(_("Stopped"))
self.seek_bar.clear()
@idle_task
def state_play_pause(self):
self.media_engine.state_play_pause()
bean = self.media_engine.bean
if self.media_engine.get_state() == STATE_PLAY:
self.statusbar.set_text(bean.info)
else:
self.statusbar.set_text(_("Paused | ") + str(bean.info))
def state_is_playing(self):
return self.media_engine.get_state() == STATE_PLAY
def fill_bean_from_vk(self, bean):
if bean.type and bean.type == FTYPE_RADIO:
return False
vk = self.vk_service.find_one_track(bean.get_display_name())
if vk:
bean.path = vk.path
bean.time = vk.time
return True
else:
return False
def fill_bean_by_vk_aid(self, bean):
if not bean.vk_audio_id:
return False
if bean.type and bean.type == FTYPE_RADIO:
return False
track = self.vk_service.find_track_by_id(bean.vk_audio_id)
if track:
bean.path = track.path
bean.time = track.time
return True
return False
@idle_task
def play(self, bean):
if not bean or not bean.is_file:
return
self.play_lock.acquire()
self.seek_bar.clear()
## TODO: Check for GTK+3.4 (Status icon doesn't have a set_tooltip method)
self.statusbar.set_text(bean.info)
self.trayicon.set_text(bean.text)
#self.movie_window.set_text(bean.text)
if bean.type == FTYPE_RADIO:
self.record.show()
self.seek_bar.progressbar.set_fraction(0)
self.seek_bar.set_text(_("Radio ") + bean.text.capitalize())
else:
self.record.hide()
self.main_window.set_title(bean.text)
thread.start_new_thread(self._one_thread_play, (bean,))
def _one_thread_play(self, bean):
try:
self._play(bean)
finally:
if self.play_lock.locked():
self.play_lock.release()
def _play(self, bean):
if not bean.path:
bean.path = get_bean_posible_paths(bean)
if not self.check_path(bean.path):
if bean.iso_path and os.path.exists(bean.iso_path):
logging.info("Try to remount " + bean.iso_path)
mount_tmp_iso(bean.iso_path)
elif bean.vk_audio_id:
self.fill_bean_by_vk_aid(bean)
elif not bean.path or ("userapi" in bean.path) or ("vk.me" in bean.path):
self.fill_bean_from_vk(bean)
else:
resource = bean.path if bean.path else bean.text
logging.error("Resourse " + resource + " not found")
self.media_engine.state_stop(show_in_tray=False)
self.statusbar.set_text(_("Resource not found"))
self.seek_bar.set_text(_("Resource not found"))
self.count_errors += 1
time.sleep(2)
if self.count_errors < 4:
if self.play_lock.locked():
self.play_lock.release()
self.next()
else:
self.seek_bar.set_text(_("Stopped. No resources found"))
return
elif os.path.isdir(bean.path):
return
self.count_errors = 0
self.media_engine.play(bean)
self.is_scrobbled = False
self.start_time = False
if bean.type != FTYPE_RADIO:
self.update_info_panel(bean)
self.set_visible_video_panel(False)
@idle_task
def notify_playing(self, pos_sec, dur_sec, bean):
if not bean.type or bean.type != FTYPE_RADIO:
self.seek_bar.update_seek_status(pos_sec, dur_sec)
else:
self.seek_bar.fill_seekbar()
if pos_sec == 2 or (pos_sec > 2 and (pos_sec % 20) == 0):
self.net_wrapper.execute(self.lastfm_service.report_now_playing, bean)
if not self.start_time:
self.start_time = str(int(time.time()))
if not self.is_scrobbled and bean.type != FTYPE_RADIO:
## song should be scrobbled if 90% has been played or played greater than 5 minutes
if pos_sec > (dur_sec * 0.9) or pos_sec > (60 * 5):
self.is_scrobbled = True
self.net_wrapper.execute(self.lastfm_service.report_scrobbled, bean, self.start_time, dur_sec)
"""download music"""
if FC().automatic_online_save and bean.path and bean.path.startswith("http://"):
self.dm.append_task(bean)
@idle_task
def notify_title(self, bean, raw_text):
logging.debug("Notify title: " + raw_text)
text = raw_text.partition("||")[0]
if not self.cache_text:
self.cache_text = text
self.statusbar.set_text(raw_text.replace("||", "|"))
text = normalize_text(text)
self.seek_bar.set_text(text)
t_bean = bean.create_from_text(text)
self.update_info_panel(t_bean)
self.set_dbus_state(STATE_PLAY, t_bean)
if FC().enable_radio_scrobbler:
start_time = str(int(time.time()))
self.net_wrapper.execute(self.lastfm_service.report_now_playing, t_bean)
if " - " in text and self.cache_text != text:
c_bean = copy.copy(bean)
prev_bean = c_bean.create_from_text(self.cache_text)
self.net_wrapper.execute(self.lastfm_service.report_scrobbled, prev_bean, start_time, 200)
self.cache_text = text
@idle_task
def notify_error(self, msg):
logging.error("notify error " + msg)
self.seek_bar.set_text(msg)
self.perspectives.get_perspective('info').clear()
@idle_task
def notify_eos(self):
self.next()
@idle_task
def player_seek(self, percent):
self.media_engine.seek(percent)
@idle_task
def player_volume(self, percent):
self.media_engine.volume(percent)
def search_vk_page_tracks(self, vk_ulr):
logging.debug("Search vk_service page tracks")
results = self.vk_service.find_tracks_by_url(vk_ulr)
all = []
p_bean = FModel(vk_ulr).add_font("bold")
all.append(p_bean)
for i, bean in enumerate(results):
bean.tracknumber = i + 1
bean.parent(p_bean).add_is_file(True)
all.append(bean)
self.notetabs.append_tab(vk_ulr, all)
def search_all_tracks(self, query):
def search_all_tracks_task():
analytics.action("SEARCH_search_all_tracks")
results = self.vk_service.find_tracks_by_query(query)
if not results:
results = []
all = []
p_bean = FModel(query).add_font("bold")
all.append(p_bean)
for i, bean in enumerate(results):
bean.tracknumber = i + 1
bean.parent(p_bean).add_is_file(True)
all.append(bean)
if not results:
all = self.show_google_results(query)
self.notetabs.append_tab(query, all)
self.in_thread.run_with_spinner(search_all_tracks_task, no_thread=True)
def search_top_tracks(self, query):
def search_top_tracks_task(query):
analytics.action("SEARCH_search_top_tracks")
results = self.lastfm_service.search_top_tracks(query)
if not results:
results = []
all = []
parent_bean = FModel(query)
all.append(parent_bean)
for i, bean in enumerate(results):
bean.tracknumber = i + 1
bean.parent(parent_bean).add_is_file(True)
all.append(bean)
if not results:
all = self.show_google_results(query)
self.notetabs.append_tab(query, all)
self.in_thread.run_with_spinner(search_top_tracks_task, query)
def search_top_albums(self, query):
def search_top_albums_task(query):
analytics.action("SEARCH_search_top_albums")
results = self.lastfm_service.search_top_albums(query)
if not results:
results = []
self.notetabs.append_tab(query, None)
albums_already_inserted = []
for album in results[:15]:
all = []
if album.album.lower() in albums_already_inserted:
continue
album.is_file = False
tracks = self.lastfm_service.search_album_tracks(album.artist, album.album)
for i, track in enumerate(tracks):
track.tracknumber = i + 1
track.album = album.album
track.parent(album).add_is_file(True)
all.append(track)
if len(all) > 0:
all = [album] + all
albums_already_inserted.append(album.album.lower())
self.notetabs.append_all(all)
if not results:
all = self.show_google_results(query)
self.notetabs.append_all(all)
self.in_thread.run_with_spinner(search_top_albums_task, query)
def search_top_similar(self, query):
def search_top_similar_task(query):
analytics.action("SEARCH_search_top_similar")
results = self.lastfm_service.search_top_similar_artist(query)
if not results:
results = []
self.notetabs.append_tab(query, None)
for artist in results[:15]:
all = []
artist.is_file = False
all.append(artist)
tracks = self.lastfm_service.search_top_tracks(artist.artist)
for i, track in enumerate(tracks):
track.tracknumber = i + 1
track.parent(artist).add_is_file(True)
all.append(track)
self.notetabs.append_all(all)
if not results:
all = self.show_google_results(query)
#inline(query)
self.in_thread.run_with_spinner(search_top_similar_task, query)
def search_top_tags(self, query):
def search_top_tags_task(query):
analytics.action("SEARCH_search_top_tags")
results = self.lastfm_service.search_top_tags(query)
if not results:
logging.debug("tag result not found")
results = []
self.notetabs.append_tab(query, None)
for tag in results[:15]:
all = []
tag.is_file = False
all.append(tag)
tracks = self.lastfm_service.search_top_tag_tracks(tag.text)
for i, track in enumerate(tracks):
track.tracknumber = i + 1
track.parent(tag).add_is_file(True)
all.append(track)
self.notetabs.append_all(all)
if not results:
all = self.show_google_results(query)
self.notetabs.append_all(all)
#inline(query)
self.in_thread.run_with_spinner(search_top_tags_task, query)
@idle_task
def update_info_panel(self, bean):
self.perspectives.get_perspective('info').update(bean)
@idle_task
def append_to_new_notebook(self, text, beans, optimization=False):
self.notetabs._append_tab(text, beans, optimization)
@idle_task
def append_to_current_notebook(self, beans):
self.notetabs.append_all(beans)
@idle_task
def next(self):
bean = self.notetabs.next()
if not bean:
return
gap = FC().gap_secs
time.sleep(gap)
logging.debug("play current bean is %s" % str(bean.text))
self.play(bean)
@idle_task
def prev(self):
bean = self.notetabs.prev()
if not bean:
return
self.play(bean)
def quit(self, *a):
self.state_stop()
self.main_window.hide()
self.trayicon.hide()
logging.info("Controls - Quit")
for element in self.__dict__:
if isinstance(self.__dict__[element], Quitable):
self.__dict__[element].on_quit()
FC().save()
GLib.idle_add(Gtk.main_quit) # wait for complete stop task
def check_version(self):
uuid = FCBase().uuid
current_version = FOOBNIX_VERSION
system = "not_set"
try:
import platform
system = platform.system()
except:
pass
try:
from socket import gethostname
f = urlopen("http://www.foobnix.com/version?uuid=" + uuid + "&host=" + gethostname()
+ "&version=" + current_version + "&platform=" + system, timeout=7)
#f = urllib2.urlopen("http://localhost:8080/version?uuid=" + uuid + "&host=" + gethostname() + "&v=" + current_version)
except Exception as e:
logging.error("Check version error: " + str(e))
return None
new_version_line = f.read()
logging.info("version " + current_version + "|" + new_version_line + "|" + str(uuid))
f.close()
if FC().check_new_version and compare_versions(current_version, new_version_line) == 1:
info_dialog_with_link_and_donate(new_version_line)
def on_load(self):
"""load controls"""
for element in self.__dict__:
if isinstance(self.__dict__[element], LoadSave):
init = time.time()
self.__dict__[element].on_load()
logging.debug("%f LOAD ON START %s" % (time.time() - init, str(self.__dict__[element])))
"""load others"""
#self.movie_window.hide_all()
self.main_window.show()
self.search_progress.stop()
"""base layout"""
self.layout.on_load()
"""check for new version"""
if os.name == 'nt':
self.check_version()
else:
pass
#GLib.idle_add(self.check_version)
@idle_task_priority(GLib.PRIORITY_LOW)
def play_first_file_in_playlist(self):
active_playlist_tree = self.notetabs.get_current_tree()
filter_model = active_playlist_tree.get_model()
current_model = filter_model.get_model()
def play_item(iter, active_playlist_tree, filter_model, current_model):
bean = active_playlist_tree.get_bean_from_model_iter(current_model, iter)
if not bean:
return
if bean.font != 'bold':
self.play(bean)
tree_selection = active_playlist_tree.get_selection()
filter_iter = filter_model.convert_child_iter_to_iter(iter)
if filter_iter[0]:
GLib.idle_add(tree_selection.select_iter, filter_iter[1])
active_playlist_tree.set_play_icon_to_bean_to_selected()
else:
iter = current_model.iter_next(iter)
play_item(iter, active_playlist_tree, filter_model, current_model)
iter = current_model.get_iter_first()
play_item(iter, active_playlist_tree, filter_model, current_model)
def on_save(self):
for element in self.__dict__:
if isinstance(self.__dict__[element], LoadSave):
logging.debug("SAVE " + str(self.__dict__[element]))
self.__dict__[element].on_save()
def download(self):
self.dm.append_task(bean=self.notetabs.get_current_tree().get_current_bean_by_UUID())
class NaptConnection(object):
def __init__(self, client, server):
Utils.expects_type(socket.socket, client, 'client')
Utils.expects_type(socket.socket, server, 'server', True)
self.lock = Lock()
self.id = 0
self.client = NaptSocket(self, client, True)
self.server = NaptSocket(self, server, False)
self.is_connecting = False
self.is_connected = False
self.is_closed = False
self.tag = None
self.tls = False
self.debug = True
self.bind_port = 0
self.connected = Event()
self.closed = Event()
self.client_closing = Event()
self.server_closing = Event()
self.client_closed = Event()
self.server_closed = Event()
self.client_recieved= Event()
self.server_recieved= Event()
def __str__(self):
return 'NaptConnection{ %s }' % ', '.join([
'id=%d' % self.id,
'client=%s' % str(self.client),
'server=%s' % str(self.server),
'is_connecting=%s' % str(self.is_connecting),
'is_connected=%s' % str(self.is_connected)])
# public
def connect(self, endpoint):
Utils.assertion(self.lock.locked(), 'need lock')
if self.is_connecting:
raise Exception() # InvalidOperationException
self.is_connecting = True
self.server.status = NaptSocketStatus.Connecting
threading.Thread(target = self.do_connect, args = (endpoint,), name = self.__class__.__name__).start()
# private
def do_connect(self, endpoint):
try:
self.server.connect(endpoint) # blocking
#self.client.socket.settimeout(5.0)
#self.server.socket.settimeout(5.0)
with self.lock:
if self.is_closed:
# todo close
return
self.is_connected = True
#print('INVOKE: on_connected')
self.on_connected(None)
except Exception as ex:
print(' endpoint: %s' % str(endpoint))
Utils.print_exception(ex)
self.close()
# public
def close(self):
with self.lock:
#if self.is_closed:
# return
self.close_client()
self.close_server()
self.is_closed = True
if self.debug:
print('NaptConnection.close: %s' % str(self))
self.on_closed(None)
# protected virtual
def on_connected(self, e):
self.connected(self, e)
# protected virtual
def on_closed(self, e):
self.closed(self, e)
# protected virtual
def on_client_closing(self, e):
self.client_closing(self, e)
# protected virtual
def on_server_closing(self, e):
self.server_closing(self, e)
# protected virtual
def on_client_closed(self, e):
self.client_closed(self, e)
# protected virtual
def on_server_closed(self, e):
self.server_closed(self, e)
# protected virtual
def on_client_recieved(self, e): # NaptConnectionEventArgs
self.client_recieved(self, e)
# protected virtual
def on_server_recieved(self, e): # NaptConnectionEventArgs
self.server_recieved(self, e)
# internal
def recv(self, so):
Utils.expects_type(NaptSocket, so, 'so')
if so.is_client:
self.recv_client()
else:
self.recv_server()
# internal
def error(self, so):
Utils.expects_type(NaptSocket, so, 'so')
# todo error
# private
def recv_client(self):
try:
#self.client.socket.settimeout(5.0)
#self.server.socket.settimeout(5.0)
data= self.client.socket.recv(4096)
e = NaptConnectionEventArgs(self, data, 0, len(data))
if len(data) == 0: # closed
#self.close_client();
self.close()
return
#print(' DATA: %s' % str(data), flush=True)
self.on_client_recieved(e)
self.server.push(data, 0, len(data))
except Exception as ex: # SocketException
Utils.print_exception(ex)
self.close()
# private
def recv_server(self):
try:
#self.client.socket.settimeout(5.0)
#self.server.socket.settimeout(5.0)
data= self.server.socket.recv(4096)
e = NaptConnectionEventArgs(self, data, 0, len(data))
if len(data) == 0: # closed
#self.close_server()
self.close()
return
#print(' DATA: %s' % str(data), flush=True)
self.on_server_recieved(e)
self.client.push(data, 0, len(data))
except Exception as ex: # SocketException
Utils.print_exception(ex)
self.close()
# private
def close_client(self):
#if self.debug:
# print(' NaptConnection.close_client: %s' % str(self.client))
try:
self.on_client_closing(None)
except Exception as ex:
Utils.print_exception(ex)
try:
if self.client.close():
self.on_client_closed(None)
except Exception as ex:
Utils.print_exception(ex)
# private void
def close_server(self):
#if self.debug:
# print(' NaptConnection.close_server: %s' % str(self.server))
try:
self.on_server_closing(None)
except Exception as ex:
Utils.print_exception(ex)
try:
if self.server.close():
self.on_server_closed(None);
except Exception as ex:
Utils.print_exception(ex)
class ConfigDialog(object):
__xml = None
__dialog = None
__mode = 0
__button = BUTTON_1
MODES = [u'0', u'A', u'B', u'AB']
__modes = []
__buttons = []
__actions = []
__act = None
__cbAct = None
__vbActConfig = None
__ulock = None
def __init__(self, config):
self.__conf = config
self.__xml = glade.XML(resource('config.glade'))
self.__xml.signal_autoconnect(self)
self.__dialog = self.__xml.get_widget('gdConfig')
self.__buttons = [self.__xml.get_widget('b%i'%(btn+1)) for btn in range(BUTTON_1, BUTTON_9+1)]
self.__modes = [None, self.__xml.get_widget('tbModeA'), self.__xml.get_widget('tbModeB')]
self.__actions = sorted(list(actionObjs()), (lambda a,b: cmp(a.label, b.label)))
self.__cbAct = self.__xml.get_widget('cbAction')
set_model_from_list(self.__cbAct, map((lambda a: a.label), self.__actions))
self.__vbActConfig = self.__xml.get_widget('vbActConfig')
self.__ulock = Lock()
self.__set_action(self.__conf[self.__mode, self.__button][0])
self.__update(True)
def __name(self):
FORMAT = u"%(num)i (%(mode)s)"
mode = self.MODES[self.__mode]
return FORMAT % {'mode': mode, 'num': self.__button + 1}
def __set_action(self, action):
if self.__act is not None:
for name, label, conf in self.__act.config:
conf.destroy()
if isinstance(action, int):
i = action
action = self.__actions[i]
else:
print "change", self.__actions, action
i = self.__actions.index(action)
self.__cbAct.set_active(i)
self.__act = action
self.__vbActConfig.forall(self.__vbActConfig.remove) # Get rid of previous children
ops = self.__conf[self.__mode, self.__button][1]
if ops is None: ops = {}
print "load ops", ops
for name, label, conf in action.config:
print "nlc:", name, label, conf
ui = conf.createInput(self.__save)
if name in ops:
conf.setValue(ops[name])
self.__vbActConfig.add(ui)
self.__vbActConfig.show_all()
def __save(self):
if self.__ulock.locked(): return # Don't save when updating
ops = {}
for name, label, conf in self.__act.config:
ops[name] = conf.getValue()
print "Save:", self.__act, ops
self.__conf[self.__mode, self.__button] = (self.__act, ops)
print "saved:", self.__conf[self.__mode, self.__button]
def __update(self, first=False):
if not self.__ulock.acquire(False): return
try:
self.__xml.get_widget('lbldButton').set_label(self.__name())
if bool(self.__mode & MODE_A) != bool(self.__modes[MODE_A].get_active()):
self.__modes[MODE_A].set_active(bool(num & MODE_A))
if bool(self.__mode & MODE_B) != bool(self.__modes[MODE_B].get_active()):
self.__modes[MODE_B].set_active(bool(self.__mode & MODE_B))
conf = self.__conf[self.__mode, self.__button]
self.__set_action(conf[0] or Action())
finally:
self.__ulock.release()
def buttonChange(self, widget, data=None):
num = self.__buttons.index(widget)
print "buttonChange", widget, num
self.__save()
if num != self.__button:
self.__button = num
self.__update()
def modeChange(self, widget, data=None):
num = self.__modes.index(widget)
print "modeChange", widget, num
self.__save()
self.__mode = (self.__mode & ~num) | (num if widget.get_active() else 0)
self.__update()
def actionChange(self, widget, data=None):
print "actionChange", widget, data
i = widget.get_active()
self.__set_action(i)
self.__save()
def __del__(self):
self.__dialog.destroy()
self.__dialog = None
def run(self):
rv = self.__dialog.run()
self.__dialog.hide()
return rv
def grab(self):
return self.__dialog.present()
class RSS2AlarmManager(object):
implements(IRSS2Document)
adapts(IAlarmManager)
def __new__(klass, manager):
syndicator_name = '__' + klass.__name__
syndicator = getattr(manager, syndicator_name, None)
if syndicator is None:
print 'Creating new RSS2AlarmManager'
syndicator = super(RSS2AlarmManager, klass).__new__(klass)
setattr(manager, syndicator_name, syndicator)
syndicator.initialize(manager)
return syndicator
def initialize(self, manager):
self.debug = 0
self.manager = manager
self.cache_lock = Lock()
self.cache_ttl = 30
self.closed_event_ttl = 3 * 60
self.caches = {}
self.close_events = {}
self.event_queue = EventQueue()
self.hostname = AlarmEvent.LOCALORIGIN
self.uri_base = 'http://' + self.hostname
if self.uri_base[-1] == '/':
self.uri_base = self.uri_base[0:-1]
self.categories = {
'RAISED': rss.Category("Raised"),
'INACTIVE': rss.Category("Inactive"),
'ACCEPTED': rss.Category("Accepted"),
'CLEARED': rss.Category("Cleared"),
'CLOSED': rss.Category("Closed")
}
self.last_pub_time = None
self.updatesub = self.manager.register_for_type(
self.initialize_caches, FormationUpdated)
self.subscription = self.manager.register_for_type(
self.handle_alarm_update, StateEvent, None, True)
self.initialize_caches()
def initialize_caches(self, *args):
if self.debug: tstart = time.time()
print 'RSS2 Syndic initializing caches because: %s' % (args,)
events = []
self.cache_lock.acquire()
try:
self.event_queue.popqueue()
for group in map(Alarm.get_events, self.manager.get_alarms()):
events.extend(group)
events.extend(self.manager.get_remote_events())
events.extend(self.event_queue.popqueue())
self.caches = {None: ItemCache()}
cache = self.process_events(events)
finally:
self.cache_lock.release()
if self.debug:
tend = time.time()
tlapse = tend - tstart
print 'RSS2 cache init of %s events took %s seconds.' % (len(events), tlapse)
return cache
def process_events(self, events):
if not self.cache_lock.locked():
raise Exception('Process events cannot be called unless locked.')
cache = {}
if events:
guids = map(Event.get_guid, events)
items = map(self.item_from_event, events)
cache.update(zip(guids, items))
for existing in self.caches.values():
existing.update(cache)
return cache
def handle_alarm_update(self, event):
if self.debug:
tstart = time.time()
if isinstance(event, StateEvent):
event = event.source
if event.is_state('closed'):
self.close_events[event.GUID] = uptime.secs()
self.event_queue.enqueue(event)
if self.cache_lock.acquire(0):
try:
self.trim_expired_caches()
self.process_events(self.event_queue.popqueue())
finally:
self.cache_lock.release()
else:
print 'Alarm update not processing queue; locked.'
if self.debug:
tend = time.time()
tlapse = tend - tstart
print 'Took RSS2 Syndic %s secs to handle alarm event.' % tlapse
return
def render(self, request_path = None, cache_id = None):
if request_path is None:
request_path = '/syndication'
xmldoc = self.setup_xmldoc()
channel = self.setup_channel(request_path)
xmldoc.root_element.channel = channel
self.cache_lock.acquire()
try:
self.trim_expired_caches()
queue = self.event_queue.popqueue()
if queue:
self.process_events(queue)
finally:
self.cache_lock.release()
items = self.get_items(cache_id)
map(channel.items.append, items)
return str(xmldoc)
def setup_xmldoc(self):
xmldoc = rss.XMLDoc()
xmldoc.root_element = rss.RSS()
return xmldoc
def setup_channel(self, request_path):
publish_time = time.time()
channel = rss.Channel()
channel.title = rss.Title("Network Building Mediator Alarms")
if request_path[0] != '/':
request_path = '/' + request_path
url = self.uri_base + request_path
channel.link = rss.Link(url)
channel.description = rss.Description("RSS 2.0 feed of Network "
"Building Mediator alarms.")
if self.last_pub_time is not None:
channel.last_build_date = rss.LastBuildDate(self.last_pub_time)
self.last_pub_time = publish_time
channel.generator = rss.Generator("Network Building "
"Mediator Alarm Syndication")
channel.docs = rss.Docs('http://blogs.law.harvard.edu/tech/rss')
return channel
def get_items(self, cid):
"""
Get Alarm Event Items that have not been returned to
client with ID "cid". If client ID is None, return all.
"""
count = None
if cid:
count = 250
if cid not in self.caches:
self.caches[cid] = ItemCache(self.caches[None])
cache = self.caches[cid]
return cache.read(count)
def trim_expired_caches(self):
if not self.cache_lock.locked():
raise Exception('Must be locked to trim caches.')
removed = []
now = uptime.secs()
allitems = self.caches[None]
for guid,closed in self.close_events.items():
if (now - closed) > self.closed_event_ttl:
if guid in allitems:
del(allitems[guid])
del(self.close_events[guid])
for cid,cache in self.caches.items():
if cid and (cache.since_touched() > self.cache_ttl):
del(self.caches[cid])
removed.append(cid)
if self.debug and removed:
print 'Cache trim trimmed the following IDs: %s.' % (removed,)
return removed
def item_from_event(self, event):
if isinstance(event, StateEvent):
event = event.source
return IRSS2ItemElement(event).as_item()
class Downloader:
def __init__(self, url, dlhash, rawserver, failed_func, max_errors = 10):
if DEBUG:
log('dd-downloader::__init__: url', url, 'hash', binascii.hexlify(dlhash))
self.url = url
self.rawserver = rawserver
self.failed_func = failed_func
self.final_url = None
self.storage = None
self.lock = Lock()
self.measure = Measure(10.0)
self.errors = 0
self.max_errors = max_errors
self.seek = None
self.shutdown_flag = False
self.running = False
self.log_prefix = 'dd-downloader::' + binascii.hexlify(dlhash) + ':'
def predownload(self, callback, timeout = 10):
if self.lock.locked():
self.seek = pos
return
t = Thread(target=self._predownload, args=[callback, timeout])
t.setName('dd-downloader-predownload-' + t.getName())
t.setDaemon(True)
t.start()
def _predownload(self, callback, timeout):
self.lock.acquire()
self.running = True
try:
if DEBUG:
log(self.log_prefix + '_predownload: url', self.url, 'timeout', timeout)
stream = urlOpenTimeout(self.url, timeout=timeout)
content_type = stream.info().getheader('Content-Type')
content_length = stream.info().getheader('Content-Length')
if DEBUG:
log(self.log_prefix + '_predownload: request finished: content_type', content_type, 'content_length', content_length)
data = ''
while True:
if self.shutdown_flag:
if DEBUG:
log(self.log_prefix + '_predownload: got shutdown flag while reading: url', self.url)
break
buf = stream.read(524288)
if not buf:
if DEBUG:
log(self.log_prefix + '_predownload: eof: url', self.url)
break
data += buf
if DEBUG:
log(self.log_prefix + '_predownload: read chunk: url', self.url, 'read_len', len(data))
stream.close()
if not self.shutdown_flag:
if DEBUG:
log(self.log_prefix + '_predownload: finished, run callback: url', self.url, 'content_type', content_type, 'content_length', content_length, 'data_len', len(data))
callback(content_type, data)
except Exception as e:
if DEBUG:
print_exc()
self.failed_func(e)
finally:
self.running = False
self.lock.release()
def init(self, callback = None, timeout = 10):
if callback is None:
return self._init()
t = Thread(target=self._init, args=[callback, timeout])
t.setName('dd-downloader-init-' + t.getName())
t.setDaemon(True)
t.start()
def _init(self, callback = None, timeout = 10):
try:
scheme, host, path = self.parse_url(self.url)
redirects = 0
connection = HTTPConnection(host)
while True:
connection.request('HEAD', path, None, {'Host': host,
'User-Agent': USER_AGENT})
r = connection.getresponse()
if r.status == 200:
break
elif r.status == 301 or r.status == 302:
redirect_url = r.getheader('Location', None)
if DEBUG:
log(self.log_prefix + 'init: got redirect: url', self.url, 'redirect', redirect_url)
scheme, rhost, path = self.parse_url(redirect_url)
redirects += 1
if redirects > MAX_REDIRECTS:
raise Exception('Too much redirects')
if rhost != host:
connection.close()
connection = HTTPConnection(rhost)
host = rhost
else:
raise Exception('Bad http status: ' + str(r.status))
mime = r.getheader('Content-Type', None)
length = r.getheader('Content-Length', None)
connection.close()
if length is None:
raise Exception('No content-length in response')
if mime is None:
raise Exception('No content-type in response')
length = int(length)
self.final_url = scheme + '://' + host + path
if DEBUG:
log(self.log_prefix + 'init: got response: length', length, 'mime', mime, 'final_url', self.final_url)
if callback is None:
return (length, mime)
callback(length, mime)
except Exception as e:
if DEBUG:
print_exc()
if callback is None:
raise e
else:
self.failed_func(e)
def set_storage(self, storage):
self.storage = storage
def start(self, pos = 0):
if self.storage is None:
raise Exception('Storage is not set')
if self.final_url is None:
raise Exception('Final url is not set')
if self.lock.locked():
self.seek = pos
return
t = Thread(target=self._request, args=[pos])
t.setName('dd-downloader-' + t.getName())
t.setDaemon(True)
t.start()
def _request(self, pos):
self.lock.acquire()
self.running = True
try:
while True:
if self.shutdown_flag:
if DEBUG:
log(self.log_prefix + '_request: got shutdown flag before read: url', self.url)
break
pos = self.storage.get_unfinished_pos(pos)
if pos is None:
if DEBUG:
log(self.log_prefix + '_request: no unfinished pos, break: url', self.url)
break
self._read(pos)
if self.seek is not None:
pos = self.seek
self.seek = None
continue
break
except ReadErrorException:
if DEBUG:
log(self.log_prefix + '_request: read error, retry immediatelly: url', self.url, 'pos', pos)
start_lambda = lambda : self.start(pos)
self.rawserver.add_task(start_lambda, 0.1)
except FatalErrorException as e:
if DEBUG:
log(self.log_prefix + '_request: fatal error, exit: url', self.url, 'pos', pos)
self.failed_func(e)
except Exception as e:
self.errors += 1
if DEBUG:
print_exc()
if self.errors > self.max_errors:
if DEBUG:
log(self.log_prefix + '_request: non-fatal error, max errors reached: errors', self.errors, 'max', self.max_errors)
self.failed_func(e)
else:
retry_in = 5 * (1 + self.errors / 10)
if DEBUG:
log(self.log_prefix + '_request: non-fatal error: url', self.url, 'pos', pos, 'errors', self.errors, 'retry_in', retry_in)
start_lambda = lambda : self.start(pos)
self.rawserver.add_task(start_lambda, retry_in)
finally:
self.running = False
self.lock.release()
def is_running(self):
return self.running
def _read(self, pos):
scheme, host, path = self.parse_url(self.final_url)
request_range = str(pos) + '-'
connection = HTTPConnection(host)
connection.request('GET', path, None, {'Host': host,
'User-Agent': USER_AGENT,
'Range': 'bytes=%s' % request_range})
r = connection.getresponse()
if DEBUG:
log(self.log_prefix + '_read: url', self.url, 'final', self.final_url, 'pos', pos, 'status', r.status)
if r.status != 200 and r.status != 206:
if DEBUG:
log(self.log_prefix + '_read: bad http status: url', self.url, 'status', r.status)
connection.close()
if 400 <= r.status < 500:
raise FatalErrorException, 'http status ' + str(r.status)
else:
raise NonFatalErrorException, 'http status ' + str(r.status)
request_size = r.getheader('Content-Length', None)
if request_size is None:
if DEBUG:
log(self.log_prefix + '_read: missing content length: url', self.url)
connection.close()
return
try:
request_size = int(request_size)
except:
if DEBUG:
print_exc()
connection.close()
return
if DEBUG:
log(self.log_prefix + '_read: url', self.url, 'request_range', request_range, 'request_size', request_size)
total_read = 0
read_size = 16384
while True:
chunk = r.read(read_size)
if not chunk:
if total_read != request_size:
if DEBUG:
log(self.log_prefix + '_read: no data, raise read error: url', self.url, 'pos', pos, 'total_read', total_read, 'request_size', request_size)
raise ReadErrorException()
if DEBUG:
log(self.log_prefix + '_read: no data, exit: url', self.url, 'pos', pos)
break
chunk_len = len(chunk)
total_read += chunk_len
if DEBUG:
log('>>>> ' + self.log_prefix + '_read: got chunk: pos', pos, 'chunk_len', chunk_len, 'total_read', total_read)
self.measure.update_rate(chunk_len)
if chunk_len != read_size and total_read != request_size:
if DEBUG:
log(self.log_prefix + '_read: bad data len, raise read error: url', self.url, 'pos', pos, 'total_read', total_read, 'request_size', request_size, 'chunk_len', chunk_len, 'read_size', read_size)
raise ReadErrorException()
if self.shutdown_flag:
if DEBUG:
log(self.log_prefix + '_read: got shutdown flag on read: url', self.url)
break
try:
t = time.time()
updated_len = self.storage.write(pos, chunk)
if DEBUG:
log('%%%%' + self.log_prefix + '_read: write to storage: pos', pos, 'len', chunk_len, 'time', time.time() - t)
if updated_len == 0:
if DEBUG:
log(self.log_prefix + '_read: data exists in storage: url', self.url, 'pos', pos, 'len', chunk_len, 'seek_flag', self.seek)
if self.seek is None:
self.seek = self.storage.get_unfinished_pos(pos)
if self.seek is None:
if DEBUG:
log(self.log_prefix + '_read: no unfinished data, exit: url', self.url, 'pos', pos)
break
except:
if DEBUG:
print_exc()
log(self.log_prefix + '_read: cannot write, exit: url', self.url)
raise FatalErrorException, 'cannot write to storage'
if self.seek is not None:
log(self.log_prefix + '_read: got seek: url', self.url, 'seek', self.seek)
break
pos += chunk_len
connection.close()
def parse_url(self, url):
scheme, host, path, pars, query, fragment = urlparse(url)
if scheme != 'http':
raise ValueError('Unsupported scheme ' + scheme)
if len(host) == 0:
raise ValueError('Empty host')
if len(path) == 0:
path = '/'
if len(pars) > 0:
path += ';' + pars
if len(query) > 0:
path += '?' + query
if len(fragment) > 0:
path += '#' + fragment
return (scheme, host, path)
def shutdown(self):
if DEBUG:
log(self.log_prefix + 'shutdown: ---')
self.shutdown_flag = True
class IOSXR(object):
"""
Establishes a connection with the IOS-XR device via SSH and facilitates the communication through the XML agent.
"""
_XML_SHELL = 'xml'
_XML_MODE_PROMPT = r'XML>'
_READ_DELAY = 0.1 # at least 0.1, corresponding to 600 max loops (60s timeout)
_XML_MODE_DELAY = 1 # should be able to read within one second
_ITERATOR_ID_ERROR_MSG = (
'Non supported IteratorID in Response object.'
'Turn iteration off on your XML agent by configuring "xml agent [tty | ssl] iteration off".'
'For more information refer to'
'http://www.cisco.com/c/en/us/td/docs/ios_xr_sw/iosxr_r4-1/xml/programming/guide/xl41apidoc.pdf, 7-99.'
'Please turn iteration off for the XML agent.'
)
def __init__(self,
hostname,
username,
password,
port=22,
timeout=60,
logfile=None,
lock=True):
"""
IOS-XR device constructor.
:param hostname: (str) IP or FQDN of the target device
:param username: (str) Username
:param password: (str) Password
:param port: (int) SSH Port (default: 22)
:param timeout: (int) Timeout (default: 60 sec)
:param logfile: File-like object to save device communication to or None to disable logging
:param lock: (bool) Auto-lock config upon open() if set to True, connect without locking if False
(default: True)
"""
self.hostname = str(hostname)
self.username = str(username)
self.password = str(password)
self.port = int(port)
self.timeout = int(timeout)
self.logfile = logfile
self.lock_on_connect = lock
self.locked = False
self._cli_prompt = None
self._xml_agent_locker = Lock()
def __getattr__(self, item):
"""
Dynamic getter to translate generic show commands.
David came up with this dynamic method. It takes
calls with show commands encoded in the name. I'll replace the
underscores for spaces and issues the show command on the device...
pretty neat!
non keyword params for show command:
all non keyword arguments is added to the command to allow dynamic parameters:
eg: .show_interface("GigabitEthernet0/0/0/0")
keyword params for show command:
config=True/False : set True to run show command in config mode
eg: .show_configuration_merge(config=True)
"""
def _getattr(*args, **kwargs):
cmd = item.replace('_', ' ')
for arg in args:
cmd += " %s" % arg
if kwargs.get("config"):
response = self._execute_config_show(cmd)
else:
response = self._execute_show(cmd)
match = re.search(".*(!! IOS XR Configuration.*)</Exec>", response, re.DOTALL)
if match is not None:
response = match.group(1)
return response
if item.startswith('show'):
return _getattr
else:
raise AttributeError("type object '%s' has no attribute '%s'" % (self.__class__.__name__, item))
def make_rpc_call(self, rpc_command):
"""
Allow a user to query a device directly using XML-requests.
:param rpc_command: (str) rpc command such as:
<Get><Operational><LLDP><NodeTable></NodeTable></LLDP></Operational></Get>
"""
result = self._execute_rpc(rpc_command)
return ET.tostring(result)
def open(self):
"""
Open a connection to an IOS-XR device.
Connects to the device using SSH and drops into XML mode.
"""
try:
self.device = ConnectHandler(device_type='cisco_xr',
ip=self.hostname,
port=self.port,
username=self.username,
password=self.password)
self.device.timeout = self.timeout
except NetMikoTimeoutException as t_err:
raise ConnectError(t_err.message)
except NetMikoAuthenticationException as au_err:
raise ConnectError(au_err.message)
self._cli_prompt = self.device.find_prompt() # get the prompt
self._enter_xml_mode()
def _timeout_exceeded(self, start=None, msg='Timeout exceeded!'):
if not start:
return False # reference not specified, noth to compare => no error
if time.time() - start > self.timeout:
# it timeout exceeded, throw TimeoutError
raise TimeoutError(msg, self)
return False
def _lock_xml_agent(self, start=None):
while (not self._xml_agent_locker.acquire(False)
and not self._timeout_exceeded(start, 'Waiting to acquire the XML agent!')):
# will wait here till the XML agent is ready to receive new requests
# if stays too much, _timeout_exceeded will raise TimeoutError
pass # do nothing, just wait
return True # ready to go now
def _unlock_xml_agent(self):
if self._xml_agent_locker.locked():
self._xml_agent_locker.release()
def _send_command_timing(self, command):
return self.device.send_command_timing(command,
delay_factor=self._READ_DELAY,
max_loops=self._XML_MODE_DELAY/self._READ_DELAY,
strip_prompt=False,
strip_command=False)
def _in_cli_mode(self):
out = self._send_command_timing('\n')
if not out:
return False
if self._cli_prompt in out:
return True
return False
def _enter_xml_mode(self):
self._unlock_xml_agent()
# release - other commands should not have anyway access to the XML agent
# when not in XML mode
self._lock_xml_agent() # make sure it won't collide with other parallel requests
out = self._send_command_timing(self._XML_SHELL) # send xml shell command
if '0x24319600' in out:
# XML agent is not enabled
raise ConnectError('XML agent is not enabled. Please configure `xml agent tty iteration off`!', self)
self._unlock_xml_agent()
if self.lock_on_connect:
self.lock()
def _send_command(self,
command,
delay_factor=None,
start=None,
expect_string=None,
read_output=None,
receive=False):
if not expect_string:
expect_string = self._XML_MODE_PROMPT
if read_output is None:
read_output = ''
if not delay_factor:
delay_factor = self._READ_DELAY
if not start:
start = time.time()
output = read_output
last_read = ''
if not read_output and not receive:
# because the XML agent is able to process only one single request over the same SSH session at a time
# first come first served
self._lock_xml_agent(start)
try:
max_loops = self.timeout / delay_factor
last_read = self.device.send_command_expect(command,
expect_string=expect_string,
strip_prompt=False,
strip_command=False,
delay_factor=delay_factor,
max_loops=max_loops)
output += last_read
except IOError as ioe:
if ((not last_read and self._in_cli_mode()) or
(self._cli_prompt in output and "% Invalid input detected at '^' marker." in output)):
# something happened
# e.g. connection with the XML agent died while reading
# netmiko throws error and the last output read is empty (ofc)
# and in CLI mode
#
# OR
#
# Sometimes the XML agent simply exits and all issued commands provide the following output
# (as in CLI mode)
# <?
# ^
# % Invalid input detected at '^' marker.
# RP/0/RSP1/CPU0:edge01.dus01#<xml version="1.0" encoding="UTF-8"?
# ^
# % Invalid input detected at '^' marker.
# RP/0/RSP1/CPU0:edge01.dus01#<xml version
#
# Which of course does not contain the XML and netmiko throws the not found error
# therefore we need to re-enter in XML mode
self._enter_xml_mode()
# and let's issue the command again if still got time
if not self._timeout_exceeded(start=start):
# if still got time
# reiterate the command from the beginning
return self._send_command(command,
expect_string=expect_string,
delay_factor=delay_factor)
else:
output += self._netmiko_recv() # try to read some more
if '0xa3679e00' in output:
# when multiple parallel request are made, the device throws the error:
# ERROR: 0xa3679e00 'XML Service Library' detected the 'fatal' condition
# 'Multiple concurrent requests are not allowed over the same session.
# A request is already in progress on this session.'
# we could use a mechanism similar to NETCONF and push the requests in queue and serve them sequentially
# BUT we are not able to assign unique IDs and identify the request-reply map
# so will throw an error that does not help too much :(
raise XMLCLIError('XML agent cannot process parallel requests!', self)
if not output.strip().endswith('XML>'):
if '0x44318c06' in output or (self._cli_prompt and expect_string != self._cli_prompt and \
(output.startswith(self._cli_prompt) or output.endswith(self._cli_prompt))):
# sometimes the device throws a stupid error like:
# ERROR: 0x44318c06 'XML-TTY' detected the 'warning' condition
# 'A Light Weight Messaging library communication function returned an error': No such device or address
# and the XML agent connection is closed, but the SSH connection is fortunately maintained
# OR sometimes, the device simply exits from the XML mode without any clue
# In both cases, we need to re-enter in XML mode...
# so, whenever the CLI promt is detected, will re-enter in XML mode
# unless the expected string is the prompt
self._unlock_xml_agent()
self._enter_xml_mode()
# however, the command could not be executed properly, so we need to raise the XMLCLIError exception
raise XMLCLIError('Could not properly execute the command. Re-entering XML mode...', self)
if not output.strip(): # empty output, means that the device did not start delivering the output
# but for sure is still in XML mode as netmiko did not throw error
if not self._timeout_exceeded(start=start):
return self._send_command(command, receive=True, start=start) # let's try receiving more
raise XMLCLIError(output.strip(), self)
self._unlock_xml_agent()
return str(output.replace('XML>', '').strip())
def _netmiko_recv(self):
output = ''
for tmp_output in self.device.receive_data_generator():
output += tmp_output
return output
# previous module function __execute_rpc__
def _execute_rpc(self, command_xml, delay_factor=.1):
xml_rpc_command = '<?xml version="1.0" encoding="UTF-8"?><Request MajorVersion="1" MinorVersion="0">' \
+ command_xml + '</Request>'
response = self._send_command(xml_rpc_command, delay_factor=delay_factor)
try:
root = ET.fromstring(response)
except ET.XMLSyntaxError as xml_err:
if 'IteratorID="' in response:
raise IteratorIDError(self._ITERATOR_ID_ERROR_MSG, self)
raise InvalidXMLResponse('Unable to process the XML Response from the device!', self)
if 'IteratorID' in root.attrib:
raise IteratorIDError(self._ITERATOR_ID_ERROR_MSG, self)
childs = [x.tag for x in list(root)]
result_summary = root.find('ResultSummary')
if result_summary is not None and int(result_summary.get('ErrorCount', 0)) > 0:
if 'CLI' in childs:
error_msg = root.find('CLI').get('ErrorMsg') or ''
elif 'Commit' in childs:
error_msg = root.find('Commit').get('ErrorMsg') or ''
error_code = root.find('Commit').get('ErrorCode') or ''
if error_code == '0x41866c00':
# yet another pointless IOS-XR error:
# if the config DB was changed by another process,
# while the current SSH connection is established and alive,
# we won't be able to commit and the device will throw the following error:
# 'CfgMgr' detected the 'warning' condition
# 'One or more commits have occurred from other configuration sessions since this session started
# or since the last commit was made from this session.'
# dumb.
# in this case we need to re-open the connection with the XML agent
_candidate_config = self.get_candidate_config(merge=True)
self.discard_config() # discard candidate config
try:
# exiting from the XML mode
self._send_command('exit', expect_string=self._cli_prompt)
except XMLCLIError:
pass # because does not end with `XML>`
self._enter_xml_mode() # re-entering XML mode
self.load_candidate_config(config=_candidate_config)
return self.commit_config()
elif error_code == '0x41864e00' or error_code == '0x43682c00':
# raises this error when the commit buffer is empty
raise CommitError('The target configuration buffer is empty.', self)
else:
error_msg = root.get('ErrorMsg') or ''
error_msg += '\nOriginal call was: %s' % xml_rpc_command
raise XMLCLIError(error_msg, self)
if 'CLI' in childs:
cli_childs = [x.tag for x in list(root.find('CLI'))]
if 'Configuration' in cli_childs:
output = root.find('CLI').find('Configuration').text
elif 'Exec' in cli_childs:
output = root.find('CLI').find('Exec').text
if output is None:
output = ''
elif 'Invalid input detected' in output:
raise InvalidInputError('Invalid input entered:\n%s' % output, self)
return root
# previous module function __execute_show__
def _execute_show(self, show_command):
"""
Executes an operational show-type command.
"""
rpc_command = '<CLI><Exec>{show_command}</Exec></CLI>'.format(
show_command=escape_xml(show_command)
)
response = self._execute_rpc(rpc_command)
raw_response = response.xpath('.//CLI/Exec')[0].text
return raw_response.strip() if raw_response else ''
# previous module function __execute_config_show__
def _execute_config_show(self, show_command, delay_factor=.1):
"""
Executes a configuration show-type command.
"""
rpc_command = '<CLI><Configuration>{show_command}</Configuration></CLI>'.format(
show_command=escape_xml(show_command)
)
response = self._execute_rpc(rpc_command, delay_factor=delay_factor)
raw_response = response.xpath('.//CLI/Configuration')[0].text
return raw_response.strip() if raw_response else ''
def close(self):
"""
Close the connection to the IOS-XR device.
Clean up after you are done and explicitly close the router connection.
"""
if self.lock_on_connect or self.locked:
self.unlock()
self._unlock_xml_agent()
self.device.remote_conn.close()
def lock(self):
"""
Lock the config database.
Use if Locking/Unlocking is not performaed automatically by lock=False
"""
if not self.locked:
rpc_command = '<Lock/>'
try:
self._execute_rpc(rpc_command)
except XMLCLIError:
raise LockError('Unable to enter in configure exclusive mode!', self)
self.locked = True
def unlock(self):
"""
Unlock the IOS-XR device config.
Use if Locking/Unlocking is not performaed automatically by lock=False
"""
if self.locked:
rpc_command = '<Unlock/>'
try:
self._execute_rpc(rpc_command)
except XMLCLIError:
raise UnlockError('Unable to unlock the config!', self)
self.locked = False
def load_candidate_config(self, filename=None, config=None):
"""
Load candidate confguration.
Populate the attribute candidate_config with the desired
configuration and loads it into the router. You can populate it from
a file or from a string. If you send both a filename and a string
containing the configuration, the file takes precedence.
:param filename: Path to the file containing the desired
configuration. By default is None.
:param config: String containing the desired configuration.
"""
configuration = ''
if filename is None:
configuration = config
else:
with open(filename) as f:
configuration = f.read()
rpc_command = '<CLI><Configuration>{configuration}</Configuration></CLI>'.format(
configuration=escape_xml(configuration) # need to escape, otherwise will try to load invalid XML
)
try:
self._execute_rpc(rpc_command)
except InvalidInputError as e:
self.discard_config()
raise InvalidInputError(e.message, self)
def get_candidate_config(self, merge=False, formal=False):
"""
Retrieve the configuration loaded as candidate config in your configuration session.
:param merge: Merge candidate config with running config to return
the complete configuration including all changed
:param formal: Return configuration in IOS-XR formal config format
"""
command = "show configuration"
if merge:
command += " merge"
if formal:
command += " formal"
response = self._execute_config_show(command)
match = re.search(".*(!! IOS XR Configuration.*)$", response, re.DOTALL)
if match is not None:
response = match.group(1)
return response
def compare_config(self):
"""
Compare configuration to be merged with the one on the device.
Compare executed candidate config with the running config and
return a diff, assuming the loaded config will be merged with the
existing one.
:return: Config diff.
"""
_show_merge = self._execute_config_show('show configuration merge')
_show_run = self._execute_config_show('show running-config')
diff = difflib.unified_diff(_show_run.splitlines(1)[2:-2], _show_merge.splitlines(1)[2:-2])
return ''.join([x.replace('\r', '') for x in diff])
def compare_replace_config(self):
"""
Compare configuration to be replaced with the one on the device.
Compare executed candidate config with the running config and
return a diff, assuming the entire config will be replaced.
:return: Config diff.
"""
diff = self._execute_config_show('show configuration changes diff')
return ''.join(diff.splitlines(1)[2:-2])
def commit_config(self, label=None, comment=None, confirmed=None):
"""
Commit the candidate config.
:param label: Commit comment, displayed in the commit entry on the device.
:param comment: Commit label, displayed instead of the commit ID on the device.
:param confirmed: Commit with auto-rollback if new commit is not made in 30 to 300 sec
"""
rpc_command = '<Commit'
if label:
rpc_command += ' Label="%s"' % label
if comment:
rpc_command += ' Comment="%s"' % comment
if confirmed:
if 30 <= int(confirmed) <= 300:
rpc_command += ' Confirmed="%d"' % int(confirmed)
else:
raise InvalidInputError('confirmed needs to be between 30 and 300 seconds', self)
rpc_command += '/>'
self._execute_rpc(rpc_command)
def commit_replace_config(self, label=None, comment=None, confirmed=None):
"""
Commit the candidate config to the device, by replacing the existing one.
:param comment: User comment saved on this commit on the device
:param label: User label saved on this commit on the device
:param confirmed: Commit with auto-rollback if new commit is not made in 30 to 300 sec
"""
rpc_command = '<Commit Replace="true"'
if label:
rpc_command += ' Label="%s"' % label
if comment:
rpc_command += ' Comment="%s"' % comment
if confirmed:
if 30 <= int(confirmed) <= 300:
rpc_command += ' Confirmed="%d"' % int(confirmed)
else:
raise InvalidInputError('confirmed needs to be between 30 and 300 seconds', self)
rpc_command += '/>'
self._execute_rpc(rpc_command)
def discard_config(self):
"""
Clear uncommited changes in the current session.
Clear previously loaded configuration on the device without committing it.
"""
rpc_command = '<Clear/>'
self._execute_rpc(rpc_command)
def rollback(self, rb_id=1):
"""
Rollback the last committed configuration.
:param rb_id: Rollback a specific number of steps. Default: 1
"""
rpc_command = '<Unlock/><Rollback><Previous>{rb_id}</Previous></Rollback><Lock/>'.format(rb_id=rb_id)
self._execute_rpc(rpc_command)
class Exec_lock:
"""A type of locking object for locking execution of relax."""
def __init__(self, fake_lock=False):
"""Set up the lock-like object.
@keyword fake_lock: A flag which is True will allow this object to be debugged as the locking mechanism is turned off.
@type fake_lock: bool
"""
# Store the arg.
self._fake_lock = fake_lock
# Init a threading.Lock object.
self._lock = Lock()
# The status container.
self._status = Status()
# The name and mode of the locker.
self._name = []
self._mode = []
# Script nesting level.
self._nest = 0
# Auto-analysis from script launch.
self._auto_from_script = False
# Debugging.
if self._fake_lock:
self.log = open('lock.log', 'w')
def acquire(self, name, mode='script'):
"""Simulate the Lock.acquire() mechanism.
@param name: The name of the locking code.
@type name: str
@keyword mode: The mode of the code trying to obtain the lock. This can be one of 'script' for the scripting interface or 'auto-analysis' for the auto-analyses.
@type mode: str
"""
# Debugging.
if self._status.debug:
sys.stdout.write("debug> Execution lock: Acquisition by '%s' ('%s' mode).\n" % (name, mode))
# Store the new name and mode.
self._name.append(name)
self._mode.append(mode)
# Nested locking.
if self.locked():
# Increment the nesting counter.
self._nest += 1
# Debugging.
if self._fake_lock:
self.log.write("Nested by %s (to level %s)\n" % (name, self._nest))
self.log.flush()
# Return without doing anything.
return
# Debugging.
if self._fake_lock:
self.log.write("Acquired by %s\n" % self._name[-1])
self.log.flush()
return
# Acquire the real lock.
lock = self._lock.acquire()
# Notify observers.
status = Status()
status.observers.exec_lock.notify()
# Return the real lock.
return lock
def locked(self):
"""Simulate the Lock.locked() mechanism."""
# Debugging (pseudo-locking based on _name).
if self._fake_lock:
if len(self._name):
return True
else:
return False
# Call the real method.
return self._lock.locked()
def release(self):
"""Simulate the Lock.release() mechanism."""
# Debugging.
if self._status.debug:
sys.stdout.write("debug> Execution lock: Release by '%s' ('%s' mode).\n" % (self._name[-1], self._mode[-1]))
# Pop the name and mode.
self._name.pop(-1)
self._mode.pop(-1)
# Nested locking.
if self._nest:
# Debugging.
if self._fake_lock:
self.log.write("Nested locking decrement (%s -> %s)\n" % (self._nest, self._nest-1))
self.log.flush()
# Decrement.
self._nest -= 1
# Return without releasing the lock.
return
# Debugging.
if self._fake_lock:
# Main text.
text = 'Release'
# Test suite info.
if hasattr(self, 'test_name'):
text = text + 'd by %s' % self.test_name
# Write out, flush, and exit the method.
self.log.write("%s\n\n" % text)
self.log.flush()
return
# Release the real lock.
release = self._lock.release()
# Notify observers.
status = Status()
status.observers.exec_lock.notify()
# Return the status.
return release
class Optimizer(Thread):
def __init__(self, rendis):
Thread.__init__(self)
# only when green light is unlocked will the optimization continue
self.green_light = Lock()
if not rendis:
raise RuntimeError("Renderer dispatcher is not provided")
self.rendis = rendis
self.rendis.register(self._on_renderer_reboot)
self.renderer = rendis.acquire()
renderer = self.renderer
self._optim_method = lambda *x: None
self._energy_list = []
self.set_param = renderer.set_param
self.get_param = renderer.get_param
self._target_img = None
self._target_scores = {}
self._finished_callback = lambda *args, **kwds: None
self._finished_callback_args = []
self._iter_callback = lambda *args: None
self._iter_callback_args = []
self.line_search_first = False
self._method_name = "unset"
self._eval_term_records = ()
self._eval_sum_record = None
self.result = None
self._stop_sign = False
self.stopable = False
@log.task_log
@plotting.plot_task
def run(self):
# check params
if self._target_img == None:
raise RuntimeError("Target image missing")
if len(self._energy_list) == 0:
raise RuntimeError("No energy function selected")
if self._method_name == "unset":
raise RuntimeError("Method unset")
if self.renderer == None:
raise RuntimeError("Renderer missing")
x = self.rendis.acquire().get_param()
self.renderer = self.rendis.acquire_new(
energy_terms=[n for f, n, w in self._energy_list],
penalty_terms=[self.penalty_name],
target_image=self._target_img,
atb_controls=False,
)
print "after reboot"
self.renderer.set_param(x)
print "after setting x"
# build optimizer
err_func = self._wrap_eval(self.energy_func)
if self.line_search_first:
self.line_search_init_param(err_func)
else:
pass
# wrap optimizer with green_light
self.result = self._optim_method(x=x, f=err_func)
self._finished_callback(*self._finished_callback_args)
print "finished!"
return
def line_search_init_param(self, func):
x = self.renderer.get_param()
jac = _get_jac(func=func, delta=0.005, x0=x)
search_direction = -func(x) / jac(x)
res = optimize.line_search(f=func, myfprime=jac, xk=x, pk=search_direction)
alpha = res[0]
x_new = x + alpha * search_direction
self.renderer.set_param(x_new)
def _wrap_eval(self, func):
"""
wrapping the evaluation function to be controllable by locks
"""
def wrapped(*x):
self.green_light.acquire()
self.green_light.release()
return func(*x)
return wrapped
def switch(self):
if self.green_light.locked():
self.green_light.release()
else:
self.green_light.acquire()
def play(self):
if self.green_light.locked():
self.green_light.release()
def pause(self):
if not self.green_light.locked():
self.green_light.acquire()
"""
method_dic is a mapping between optimizing method to a closure that takes
the name (in some case like CMA, name doesn't affect anything) and returns
the uniformed optimizing interface fmin(f, x0)
"""
method_dic = {
"CMA": cma_optimize,
"Nelder-Mead": scipy_optimize,
"Powell": scipy_optimize,
"CG": scipy_optimize_jac,
"BFGS": scipy_optimize_jac,
"Newton-CG": scipy_optimize_jac,
"L-BFGS-B": scipy_optimize_jac,
"TNC": scipy_optimize_jac,
"COBYLA": scipy_optimize,
"SLSQP": scipy_optimize_jac
# ,"dogleg": scipy_optimize_jac, # needs hessian, maybe later
# "trust-ncg": scipy_optimize_jac
}
linear_search_list = ["CG", "BFGS"] # TODO: finish the list
def set_method(self, name):
# this is the interface for manager
if name == "CMA":
self._optim_method = self._explict_CMA
self.stopable = True
else:
self._optim_method = Optimizer.method_dic[name](name)
self._method_name = name
def _explict_CMA(self, f, x):
# it is very inappropriate to call them solutions as they are guesses
# I'd rather call it samples or guesses but I yield
def generate(solutions):
for x in solutions:
try:
yield f(x)
except Exception as e:
# like encountering an all-black error
print e.message
self.renderer = self.rendis.acquire_new()
print "renderer reboot"
yield f(x)
sigma_0 = 1
ftarget = 1.5e-4
opts = cma.CMAOptions()
opts["ftarget"] = ftarget
es = cma.CMAEvolutionStrategy(x, sigma_0, opts)
while not es.stop() and not self._stop_sign:
solutions = es.ask()
fvals = [y for y in generate(solutions)]
es.tell(solutions, fvals)
res = es.result()
self.best_x = es.best.x
self.best_f = es.best.f
return res[0], res[1]
def stop(self):
"""
Only works under using CMAES.
Called to terminate the current optimization thread. Note that the
termination is not instant, but will wait for the last round of CMA
to finish.
"""
self._stop_sign = True
@plotting.init_plot
def set_energy(self, func_names, weights):
if len(func_names) != len(weights):
err_msg = "energy function number (%d) doesn't match weight number (%d)" % (len(func_names), len(weights))
raise RuntimeError(err_msg)
self._energy_list = zip(
[Optimizer.energy_dic[name](self._target_img, self._target_img_path) for name in func_names],
func_names,
weights,
)
self._eval_term_records = tuple([] for i in func_names)
self._eval_sum_record = []
def set_target(self, image_path):
# TODO: finish setting the target image
self._target_img_path = image_path
self._target_img = Image.open(image_path)
self._target_img = self._target_img.convert("L")
# used to have other pre-computation when setting an target image
def set_finished_callback(self, callback, *args):
self._finished_callback = callback
self._finished_callback_args = args
def set_iter_callback(self, callback, *args):
self._iter_callback = callback
self._iter_callback_args = args
# TODO: penalty could be listed
penalty_weight = 1e-3
penalty_name = "penalty"
def penalty(self, x):
res = sum(x ** 2) * self.penalty_weight
self.renderer.set_penalty_value(self.penalty_name, res)
return res
def get_mat_model2snapshot(self, img):
mat_shaj = self.renderer.shadow.shaject_mat
mat_view = self.renderer.cam_cap.view_mat
mat_proj = self.renderer.cam_cap.proj_mat
w, h = img.size
# mat_c2s is projection from clip coordinate to image coordinate
mat_c2s = np.array(
[[(w - 1) / 2, 0, 0, (w - 1) / 2], [0, (1 - h) / 2, 0, (h - 1) / 2], [0, 0, 0, 0], [0, 0, 0, 1]]
)
mat_c2s *= np.array(mat_proj * mat_view * mat_shaj).T # cgkit.mat4 is column major
return mat_c2s
def model_center_penalty_closure(self, img):
# presuming the camera for capturing, light and receiver won't move
# or otherwise all the mat shall be computed on the fly
mat_c2s = self.get_mat_model2snapshot(img)
def model_center_penalty(x):
positions = self._strip_positions(x)
return 0
return model_center_penalty
def _strip_positions(self, x):
return [tuple(x[0:3]), tuple(x[6:9]), tuple(x[12:15])]
def _on_renderer_reboot(self):
self.renderer = self.rendis.acquire()
# obsolete
def _record_term(f):
# decorator that records each term of each evaluation
@wraps(f)
def inner(cls, x):
res = f(cls, x)
for record, res_term in zip(cls._eval_term_records, res):
y, w, n = res_term
record.append(y)
return res
return inner
# obsolete
def _record_sum(f):
# decorator that records sum of all terms of each evaluation
@wraps(f)
def inner(cls, x):
res = f(cls, x)
cls._eval_sum_record.append(res)
return res
return inner
def _sum(f):
@log.energy_sum_log
@wraps(f)
def inner(cls, x):
res = f(cls, x)
for y, w, n in res:
cls.renderer.set_energy_value(name=n, val=y * w)
total = sum([w * y for y, w, n in res]) + cls.penalty(x)
cls.renderer.set_total(total)
return total
return inner
@plotting.plot_sum
@_sum
@log.energy_term_log
@plotting.plot_terms
def energy_func(self, x):
self.renderer.set_param(x)
img = self.renderer.acquire_snapshot()
self._iter_callback(*self._iter_callback_args)
return [(func(img), weight, name) for func, name, weight in self._energy_list]
"""
energy_dic is a static attribute of optimizer, which maps energy
function names to a callable instance. The mapped callable is a closure
that takes the target image as input, and return a method calculating the
energy function value based on input image.
"""
energy_dic = {
"XOR comparison": xor_closure,
"first moments (normalized)": sq_diff_closure(get_fst_moments),
"secondary moments (normalized)": sq_diff_closure(get_sec_moments),
"uncovered pixels": uncovered_closure,
"distance field": distance_field_closure,
}
class SQL_Panel(ScrollView):
'''
The logback panel is used to turn logs on and off as well as adjust the error level of each one
and the default error level.
'''
grid = ObjectProperty(None)
active = BooleanProperty(False)
query = "Select * from logs where parent in (select parent from logs where line like '%org.hibernate.SQL%') ORDER BY parent"
get_parents = "select parent from logs where line like '%org.hibernate.SQL%'"
get_exclusive_parents = "select parent from logs where line like '%org.hibernate.SQL%' and parent not in ({0})"
parent_query = "Select * from logs where parent is {0}"
def setServer(self,server):
'''
Save the local variables needed. Server specifically, but also this method spins up an
instance of the Logback model to reference.
'''
self.server = server
self.logback = model.LogbackFile(server.con)
loggers = self.logback.getLoggers()
self.loggers = []
for i in loggers:
if i.name == 'org.hibernate.SQL':
self.ohsql = i
if i.name == 'org.hibernate.type':
self.ohtype = i
self.active = not (self.ohsql.commented and self.ohtype.commented)
self.load_queries()
self.bind(on_touch_down=self.on_touch_down)
def load_queries(self):
self.logFile = model.gstore.logManager.openLog(
'/usr/local/tomcat/logs/portal/portal.log',self.server.con)
self.query_list_hashes = []
self.query_list = {}
self.query_nodes = {}
self.updating = Lock()
self.parents = []
Clock.schedule_interval(self.thread_update,1)
def thread_update(self,*args):
Logger.debug("SQL Panel: Is there a running updater thread? {0}".format(self.updating.locked()))
if self.updating.locked() == False:
Logger.debug("SQL Panel: Preparing a new thread")
t = Thread(target=self.update,args=args)
t.start()
Logger.debug("SQL Panel: Dispached thread")
Logger.debug("SQL Panel: Done checking the updater.")
def update(self, *args):
#this makes sure only one update process is running at a time.
Logger.debug("SQL Panel: Is an update process alread running?: {0}".format(self.updating.locked()))
if self.updating.acquire(False) == False:
Logger.debug("SQL Panel: Lock not acquired: {0}".format(self.updating.locked()))
return
Logger.debug("SQL Panel: Acquired lock:".format(self.updating.locked()))
#Get the list of parents into a string form.
p_list = ','.join([str(i) for i in self.parents])
#query for the queries we don't already have
parents = self.logFile.query(self.get_exclusive_parents.format(p_list))
#save those to the raw parents list. May get ride of this later.
self.parents_raw = parents
#Process the new queries:
for p in parents:
self.parents.append(p[0])
qhash, query = self.process_query(p[0])
if qhash in self.query_list_hashes:
#self.query_list[qhash].ids.append(p[0])
self.query_list[qhash].add_instance(p[0])
else:
self.query_list_hashes.append(qhash)
self.query_list[qhash] = SQL_Query(qhash,query,p[0],self)
self.updating.release()
Logger.debug("SQL Panel: Done updating, toggling self.update to: {0}".format(self.updating.locked()))
def process_query(self,parent):
'''
Take the given number, find all the parts of the query, and then string them together into
one piece. This also returns the hash for that query to make use in dictionaries easier.
'''
query_raw = self.logFile.query(self.parent_query.format(parent))
query = ''
query = '\n'.join([i[2] for i in query_raw[1:]])
q_hash = hash(query)
return (q_hash,query)
def get_from(self,query):
'''
I mostly made this to get a snippet to make the query preview short.
'''
q_split = query.split('\n')
for i,j in enumerate(q_split):
if "from" in j:
return q_split[0] + " " + j + " " + q_split[i+1] + "..."
elif "update" in j:
return j + " " + q_split[i+1] + "..."
elif "into" in j:
return j + " " + q_split[i+1] + "..."
return q_split[1]
def load_logger(self,logger):
'''
Load the logger and create it's view.
'''
self.logger = logger
self.name = logger.name
if logger.commented:
self.active = False
else:
self.active = True
self.level = logger.level
def toggle_me(self):
'''
called whenever the status of the activity toggle changes. This method makes sure that the
underlying logger get's the memo.
'''
self.active = self.ids['logback_switch'].active
if self.active:
self.ohsql.commented = False
self.ohtype.commented = False
self.logback.save()
else:
self.ohsql.commented = True
self.ohtype.commented = True
self.logback.save()
return self.active
def on_touch_down(self, touch):
if touch.is_double_tap:
try:
Clipboard.copy(self.query_tree.selected_node.text)
except AttributeError:
Logger.debug("SQL Panel: Object didn't have text.")
ScrollView.on_touch_down(self, touch)
class IMAPServer:
"""Initializes all variables from an IMAPRepository() instance
Various functions, such as acquireconnection() return an IMAP4
object on which we can operate.
Public instance variables are: self.:
delim The server's folder delimiter. Only valid after acquireconnection()
"""
GSS_STATE_STEP = 0
GSS_STATE_WRAP = 1
def __init__(self, repos):
self.ui = getglobalui()
self.repos = repos
self.config = repos.getconfig()
self.tunnel = repos.getpreauthtunnel()
self.usessl = repos.getssl()
self.username = None if self.tunnel else repos.getuser()
self.password = None
self.passworderror = None
self.goodpassword = None
self.hostname = None if self.tunnel else repos.gethost()
self.port = repos.getport()
if self.port == None:
self.port = 993 if self.usessl else 143
self.sslclientcert = repos.getsslclientcert()
self.sslclientkey = repos.getsslclientkey()
self.sslcacertfile = repos.getsslcacertfile()
if self.sslcacertfile is None:
self.verifycert = None # disable cert verification
self.delim = None
self.root = None
self.maxconnections = repos.getmaxconnections()
self.availableconnections = []
self.assignedconnections = []
self.lastowner = {}
self.semaphore = BoundedSemaphore(self.maxconnections)
self.connectionlock = Lock()
self.reference = repos.getreference()
self.idlefolders = repos.getidlefolders()
self.gss_step = self.GSS_STATE_STEP
self.gss_vc = None
self.gssapi = False
def getpassword(self):
"""Returns the server password or None"""
if self.goodpassword != None: # use cached good one first
return self.goodpassword
if self.password != None and self.passworderror == None:
return self.password # non-failed preconfigured one
# get 1) configured password first 2) fall back to asking via UI
self.password = self.repos.getpassword() or \
self.ui.getpass(self.repos.getname(), self.config,
self.passworderror)
self.passworderror = None
return self.password
def getroot(self):
"""Returns this server's folder root. Can only be called after one
or more calls to acquireconnection."""
return self.root
def releaseconnection(self, connection, drop_conn=False):
"""Releases a connection, returning it to the pool.
:param drop_conn: If True, the connection will be released and
not be reused. This can be used to indicate broken connections."""
if connection is None: return #noop on bad connection
self.connectionlock.acquire()
self.assignedconnections.remove(connection)
# Don't reuse broken connections
if connection.Terminate or drop_conn:
connection.logout()
else:
self.availableconnections.append(connection)
self.connectionlock.release()
self.semaphore.release()
def md5handler(self, response):
challenge = response.strip()
self.ui.debug('imap', 'md5handler: got challenge %s' % challenge)
passwd = self.getpassword()
retval = self.username + ' ' + hmac.new(passwd, challenge).hexdigest()
self.ui.debug('imap', 'md5handler: returning %s' % retval)
return retval
def plainauth(self, imapobj):
self.ui.debug('imap', 'Attempting plain authentication')
imapobj.login(self.username, self.getpassword())
def gssauth(self, response):
data = base64.b64encode(response)
try:
if self.gss_step == self.GSS_STATE_STEP:
if not self.gss_vc:
rc, self.gss_vc = kerberos.authGSSClientInit('imap@' +
self.hostname)
response = kerberos.authGSSClientResponse(self.gss_vc)
rc = kerberos.authGSSClientStep(self.gss_vc, data)
if rc != kerberos.AUTH_GSS_CONTINUE:
self.gss_step = self.GSS_STATE_WRAP
elif self.gss_step == self.GSS_STATE_WRAP:
rc = kerberos.authGSSClientUnwrap(self.gss_vc, data)
response = kerberos.authGSSClientResponse(self.gss_vc)
rc = kerberos.authGSSClientWrap(self.gss_vc, response,
self.username)
response = kerberos.authGSSClientResponse(self.gss_vc)
except kerberos.GSSError as err:
# Kerberos errored out on us, respond with None to cancel the
# authentication
self.ui.debug('imap', '%s: %s' % (err[0][0], err[1][0]))
return None
if not response:
response = ''
return base64.b64decode(response)
def acquireconnection(self):
"""Fetches a connection from the pool, making sure to create a new one
if needed, to obey the maximum connection limits, etc.
Opens a connection to the server and returns an appropriate
object."""
self.semaphore.acquire()
self.connectionlock.acquire()
curThread = currentThread()
imapobj = None
if len(self.availableconnections): # One is available.
# Try to find one that previously belonged to this thread
# as an optimization. Start from the back since that's where
# they're popped on.
imapobj = None
for i in range(len(self.availableconnections) - 1, -1, -1):
tryobj = self.availableconnections[i]
if self.lastowner[tryobj] == curThread.ident:
imapobj = tryobj
del(self.availableconnections[i])
break
if not imapobj:
imapobj = self.availableconnections[0]
del(self.availableconnections[0])
self.assignedconnections.append(imapobj)
self.lastowner[imapobj] = curThread.ident
self.connectionlock.release()
return imapobj
self.connectionlock.release() # Release until need to modify data
""" Must be careful here that if we fail we should bail out gracefully
and release locks / threads so that the next attempt can try...
"""
success = 0
try:
while not success:
# Generate a new connection.
if self.tunnel:
self.ui.connecting('tunnel', self.tunnel)
imapobj = imaplibutil.IMAP4_Tunnel(self.tunnel,
timeout=socket.getdefaulttimeout())
success = 1
elif self.usessl:
self.ui.connecting(self.hostname, self.port)
fingerprint = self.repos.get_ssl_fingerprint()
imapobj = imaplibutil.WrappedIMAP4_SSL(self.hostname,
self.port,
self.sslclientkey,
self.sslclientcert,
self.sslcacertfile,
self.verifycert,
timeout=socket.getdefaulttimeout(),
fingerprint=fingerprint
)
else:
self.ui.connecting(self.hostname, self.port)
imapobj = imaplibutil.WrappedIMAP4(self.hostname, self.port,
timeout=socket.getdefaulttimeout())
if not self.tunnel:
try:
# Try GSSAPI and continue if it fails
if 'AUTH=GSSAPI' in imapobj.capabilities and have_gss:
self.connectionlock.acquire()
self.ui.debug('imap',
'Attempting GSSAPI authentication')
try:
imapobj.authenticate('GSSAPI', self.gssauth)
except imapobj.error as val:
self.gssapi = False
self.ui.debug('imap',
'GSSAPI Authentication failed')
else:
self.gssapi = True
kerberos.authGSSClientClean(self.gss_vc)
self.gss_vc = None
self.gss_step = self.GSS_STATE_STEP
#if we do self.password = None then the next attempt cannot try...
#self.password = None
self.connectionlock.release()
if not self.gssapi:
if 'STARTTLS' in imapobj.capabilities and not\
self.usessl:
self.ui.debug('imap',
'Using STARTTLS connection')
imapobj.starttls()
if 'AUTH=CRAM-MD5' in imapobj.capabilities:
self.ui.debug('imap',
'Attempting CRAM-MD5 authentication')
try:
imapobj.authenticate('CRAM-MD5',
self.md5handler)
except imapobj.error as val:
self.plainauth(imapobj)
else:
# Use plaintext login, unless
# LOGINDISABLED (RFC2595)
if 'LOGINDISABLED' in imapobj.capabilities:
raise OfflineImapError("Plaintext login "
"disabled by server. Need to use SSL?",
OfflineImapError.ERROR.REPO)
self.plainauth(imapobj)
# Would bail by here if there was a failure.
success = 1
self.goodpassword = self.password
except imapobj.error as val:
self.passworderror = str(val)
raise
# update capabilities after login, e.g. gmail serves different ones
typ, dat = imapobj.capability()
if dat != [None]:
imapobj.capabilities = tuple(dat[-1].upper().split())
if self.delim == None:
listres = imapobj.list(self.reference, '""')[1]
if listres == [None] or listres == None:
# Some buggy IMAP servers do not respond well to LIST "" ""
# Work around them.
listres = imapobj.list(self.reference, '"*"')[1]
if listres == [None] or listres == None:
# No Folders were returned. This occurs, e.g. if the
# 'reference' prefix does not exist on the mail
# server. Raise exception.
err = "Server '%s' returned no folders in '%s'" % \
(self.repos.getname(), self.reference)
self.ui.warn(err)
raise Exception(err)
self.delim, self.root = \
imaputil.imapsplit(listres[0])[1:]
self.delim = imaputil.dequote(self.delim)
self.root = imaputil.dequote(self.root)
self.connectionlock.acquire()
self.assignedconnections.append(imapobj)
self.lastowner[imapobj] = curThread.ident
self.connectionlock.release()
return imapobj
except Exception as e:
"""If we are here then we did not succeed in getting a
connection - we should clean up and then re-raise the
error..."""
self.semaphore.release()
if(self.connectionlock.locked()):
self.connectionlock.release()
severity = OfflineImapError.ERROR.REPO
if type(e) == gaierror:
#DNS related errors. Abort Repo sync
#TODO: special error msg for e.errno == 2 "Name or service not known"?
reason = "Could not resolve name '%s' for repository "\
"'%s'. Make sure you have configured the ser"\
"ver name correctly and that you are online."%\
(self.hostname, self.repos)
raise OfflineImapError(reason, severity)
elif isinstance(e, SSLError) and e.errno == 1:
# SSL unknown protocol error
# happens e.g. when connecting via SSL to a non-SSL service
if self.port != 993:
reason = "Could not connect via SSL to host '%s' and non-s"\
"tandard ssl port %d configured. Make sure you connect"\
" to the correct port." % (self.hostname, self.port)
else:
reason = "Unknown SSL protocol connecting to host '%s' for"\
"repository '%s'. OpenSSL responded:\n%s"\
% (self.hostname, self.repos, e)
raise OfflineImapError(reason, severity)
elif isinstance(e, socket.error) and e.args[0] == errno.ECONNREFUSED:
# "Connection refused", can be a non-existing port, or an unauthorized
# webproxy (open WLAN?)
reason = "Connection to host '%s:%d' for repository '%s' was "\
"refused. Make sure you have the right host and port "\
"configured and that you are actually able to access the "\
"network." % (self.hostname, self.port, self.reposname)
raise OfflineImapError(reason, severity)
# Could not acquire connection to the remote;
# socket.error(last_error) raised
if str(e)[:24] == "can't open socket; error":
raise OfflineImapError("Could not connect to remote server '%s' "\
"for repository '%s'. Remote does not answer."
% (self.hostname, self.repos),
OfflineImapError.ERROR.REPO)
else:
# re-raise all other errors
raise
def connectionwait(self):
"""Waits until there is a connection available. Note that between
the time that a connection becomes available and the time it is
requested, another thread may have grabbed it. This function is
mainly present as a way to avoid spawning thousands of threads
to copy messages, then have them all wait for 3 available connections.
It's OK if we have maxconnections + 1 or 2 threads, which is what
this will help us do."""
self.semaphore.acquire()
self.semaphore.release()
def close(self):
# Make sure I own all the semaphores. Let the threads finish
# their stuff. This is a blocking method.
with self.connectionlock:
# first, wait till all connections had been released.
# TODO: won't work IMHO, as releaseconnection() also
# requires the connectionlock, leading to a potential
# deadlock! Audit & check!
threadutil.semaphorereset(self.semaphore, self.maxconnections)
for imapobj in self.assignedconnections + self.availableconnections:
imapobj.logout()
self.assignedconnections = []
self.availableconnections = []
self.lastowner = {}
# reset kerberos state
self.gss_step = self.GSS_STATE_STEP
self.gss_vc = None
self.gssapi = False
def keepalive(self, timeout, event):
"""Sends a NOOP to each connection recorded. It will wait a maximum
of timeout seconds between doing this, and will continue to do so
until the Event object as passed is true. This method is expected
to be invoked in a separate thread, which should be join()'d after
the event is set."""
self.ui.debug('imap', 'keepalive thread started')
while not event.isSet():
self.connectionlock.acquire()
numconnections = len(self.assignedconnections) + \
len(self.availableconnections)
self.connectionlock.release()
threads = []
for i in range(numconnections):
self.ui.debug('imap', 'keepalive: processing connection %d of %d' % (i, numconnections))
if len(self.idlefolders) > i:
# IDLE thread
idler = IdleThread(self, self.idlefolders[i])
else:
# NOOP thread
idler = IdleThread(self)
idler.start()
threads.append(idler)
self.ui.debug('imap', 'keepalive: waiting for timeout')
event.wait(timeout)
self.ui.debug('imap', 'keepalive: after wait')
for idler in threads:
# Make sure all the commands have completed.
idler.stop()
idler.join()
self.ui.debug('imap', 'keepalive: all threads joined')
self.ui.debug('imap', 'keepalive: event is set; exiting')
return
def verifycert(self, cert, hostname):
'''Verify that cert (in socket.getpeercert() format) matches hostname.
CRLs are not handled.
Returns error message if any problems are found and None on success.
'''
errstr = "CA Cert verifying failed: "
if not cert:
return ('%s no certificate received' % errstr)
dnsname = hostname.lower()
certnames = []
# cert expired?
notafter = cert.get('notAfter')
if notafter:
if time.time() >= cert_time_to_seconds(notafter):
return '%s certificate expired %s' % (errstr, notafter)
# First read commonName
for s in cert.get('subject', []):
key, value = s[0]
if key == 'commonName':
certnames.append(value.lower())
if len(certnames) == 0:
return ('%s no commonName found in certificate' % errstr)
# Then read subjectAltName
for key, value in cert.get('subjectAltName', []):
if key == 'DNS':
certnames.append(value.lower())
# And finally try to match hostname with one of these names
for certname in certnames:
if (certname == dnsname or
'.' in dnsname and certname == '*.' + dnsname.split('.', 1)[1]):
return None
return ('%s no matching domain name found in certificate' % errstr)
class SuccessLock():
def __init__(self, infohash = None):
self.lock = Lock()
self.pause = Lock()
self.ready = Lock()
self.code = 0L
self.success = False
self.finished = True
self.log_prefix = 'rerequester:successlock::'
if infohash is not None:
self.log_prefix += binascii.hexlify(infohash) + ':'
def start(self):
if DEBUG_LOCK:
log(self.log_prefix + 'start: acquire ready lock: thread', currentThread().name)
self.ready.acquire()
if DEBUG_LOCK:
log(self.log_prefix + 'start: acquire ready lock done: thread', currentThread().name)
self.success = False
self.finished = False
def finish(self):
if DEBUG_LOCK:
log(self.log_prefix + 'finish: release ready lock: thread', currentThread().name)
self.ready.release()
def isready(self):
locked = self.ready.locked()
if DEBUG_LOCK:
log(self.log_prefix + 'isready: ready lock status: locked', locked, 'thread', currentThread().name)
return not locked
def set(self):
if DEBUG_LOCK:
log(self.log_prefix + 'set: acquire lock: thread', currentThread().name)
self.lock.acquire()
if DEBUG_LOCK:
log(self.log_prefix + 'set: acquire lock done: thread', currentThread().name)
if not self.pause.locked():
if DEBUG_LOCK:
log(self.log_prefix + 'set: pause is not locked, acquire: thread', currentThread().name)
self.pause.acquire()
if DEBUG_LOCK:
log(self.log_prefix + 'set: pause acquire done: thread', currentThread().name)
elif DEBUG_LOCK:
log(self.log_prefix + 'set: pause is locked: thread', currentThread().name)
self.first = True
self.finished = False
self.success = False
self.code += 1L
self.lock.release()
if DEBUG_LOCK:
log(self.log_prefix + 'set: release lock and return: first', self.first, 'code', self.code, 'thread', currentThread().name)
return self.code
def trip(self, code, s = False):
if DEBUG_LOCK:
log(self.log_prefix + 'trip: acquire lock: code', code, 's', s, 'self.code', self.code, 'self.finished', self.finished, 'thread', currentThread().name)
self.lock.acquire()
if DEBUG_LOCK:
log(self.log_prefix + 'trip: acquire lock done: code', code, 's', s, 'self.code', self.code, 'self.finished', self.finished, 'thread', currentThread().name)
try:
if code == self.code and not self.finished:
r = self.first
self.first = False
if s:
self.finished = True
self.success = True
if DEBUG_LOCK:
log(self.log_prefix + 'trip: got match: code', code, 's', s, 'self.code', self.code, 'self.finished', self.finished, 'self.success', self.success, 'r', r, 'thread', currentThread().name)
return r
if DEBUG_LOCK:
log(self.log_prefix + 'trip: no match: code', code, 'self.code', self.code, 'self.finished', self.finished, 'thread', currentThread().name)
finally:
self.lock.release()
def give_up(self):
self.lock.acquire()
self.success = False
self.finished = True
if DEBUG_LOCK:
log(self.log_prefix + 'give_up: self.success', self.success, 'self.finished', self.finished, 'thread', currentThread().name)
self.lock.release()
def wait(self):
if DEBUG_LOCK:
log(self.log_prefix + 'wait: acquire pause: thread', currentThread().name)
self.pause.acquire()
if DEBUG_LOCK:
log(self.log_prefix + 'wait: acquire pause done: thread', currentThread().name)
def unwait(self, code):
if code == self.code and self.pause.locked():
if DEBUG_LOCK:
log(self.log_prefix + 'unwait: release pause: code', code, 'self.code', self.code, 'thread', currentThread().name)
self.pause.release()
elif DEBUG_LOCK:
log(self.log_prefix + 'unwait: do not release pause: code', code, 'self.code', self.code, 'thread', currentThread().name)
def isfinished(self):
self.lock.acquire()
x = self.finished
self.lock.release()
if DEBUG_LOCK:
log(self.log_prefix + 'isfinished: x', x, 'thread', currentThread().name)
return x
class GeneralLogger(object):
def __init__(self, redirect_to, job_backend=None):
self.job_backend = job_backend
self.buffer = ''
self.last_timer = None
self.last_messages = ''
self.logger = redirect_to or sys.__stdout__
self.lock = Lock()
self.attach_last_messages = {}
self.buffer_disabled = False
def disable_buffer(self):
self.buffer_disabled = True
self.buffer = ''
def clear_buffer(self):
self.buffer = ''
def fileno(self):
return self.logger.fileno()
def isatty(self):
return self.logger.isatty()
def flush(self):
self.logger.flush()
self.send_buffer()
def send_buffer(self):
self.last_timer = None
if self.buffer:
if self.job_backend:
if self.job_backend.write_log(self.buffer):
self.buffer = ''
def attach(self, buffer, read_line=None):
"""
Read buffer until end (read() returns '') and sends it to self.logger and self.job_backend.
:param buffer: a buffer instance with block read() or readline() method
:param read_line: callable or True to read line per line. If callable is given, it will be executed per line
and ignores does not redirect the line to stdout/logger when callable returns False.
"""
bid = id(buffer)
self.attach_last_messages[bid] = b''
def reader():
current_line = b''
def handle_line(buf):
if chunk == b'':
return
if read_line and callable(read_line):
res = read_line(buf)
if res is False:
return False
elif res is not None:
buf = res
if hasattr(buf, 'encode'):
buf = buf.encode('utf-8')
self.attach_last_messages[bid] += buf
if len(self.attach_last_messages[bid]) > 21 * 1024:
self.attach_last_messages[bid] = self.attach_last_messages[bid][-20 * 1024:]
self.write(buf)
flush_char = b'\n'
while True:
try:
# needs to be 1 so we fetch data in near real-time
chunk = buffer.read(1)
if chunk == b'':
if current_line:
handle_line(current_line)
return
current_line += chunk
while flush_char in current_line:
pos = current_line.find(flush_char)
line = current_line[:pos+1]
current_line = current_line[pos+1:]
handle_line(line)
# todo, periodically flush by '\r' only (progress bars for example)
# and make sure only necessary data is sent (by applying \r and \b control characters)
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
# we need to make sure, we continue to read otherwise the process of this buffer
# will block and we have a stuck process.
sys.__stderr__.write(traceback.format_exc() + '\n')
sys.__stderr__.flush()
thread = Thread(target=reader)
thread.daemon = True
thread.start()
def wait():
thread_join_non_blocking(thread)
self.send_buffer()
return wait
def write(self, message):
try:
self.lock.acquire()
if b'' == message:
return
if hasattr(message, 'decode'):
# don't decode string again
# necessary for Python3
message = message.decode('utf-8', errors='replace')
self.logger.write(message)
self.logger.flush()
self.last_messages += message
if len(self.last_messages) > 20 * 1024:
self.last_messages = self.last_messages[-20 * 1024:]
if not self.buffer_disabled:
for char in message:
if '\b' == char and self.buffer:
self.buffer = self.buffer[:-1]
else:
self.buffer += char
if not self.last_timer:
self.last_timer = Timer(1.0, self.send_buffer)
self.last_timer.start()
except (KeyboardInterrupt, SystemExit):
raise
except Exception:
sys.__stderr__.write(traceback.format_exc() + '\n')
sys.__stderr__.flush()
finally:
if self.lock.locked():
self.lock.release()
class _Selection(DOMWidget):
"""Base class for Selection widgets
``options`` can be specified as a list or dict. If given as a list,
it will be transformed to a dict of the form ``{str(value):value}``.
When programmatically setting the value, a reverse lookup is performed
among the options to set the value of ``selected_label`` accordingly. The
reverse lookup uses the equality operator by default, but an other
predicate may be provided via the ``equals`` argument. For example, when
dealing with numpy arrays, one may set equals=np.array_equal.
"""
value = Any(help="Selected value")
selected_label = Unicode(help="The label of the selected value").tag(sync=True)
options = Any(help="""List of (key, value) tuples or dict of values that the
user can select.
The keys of this list are the strings that will be displayed in the UI,
representing the actual Python choices.
The keys of this list are also available as _options_labels.
""")
_options_dict = Dict()
_options_labels = Tuple().tag(sync=True)
_options_values = Tuple()
disabled = Bool(False, help="Enable or disable user changes").tag(sync=True)
description = Unicode(help="Description of the value this widget represents").tag(sync=True)
def __init__(self, *args, **kwargs):
self.value_lock = Lock()
self.options_lock = Lock()
self.equals = kwargs.pop('equals', lambda x, y: x == y)
self.observe(self._options_readonly_changed, names=['_options_dict', '_options_labels', '_options_values', '_options'])
if 'options' in kwargs:
self.options = kwargs.pop('options')
DOMWidget.__init__(self, *args, **kwargs)
self._value_in_options()
def _make_options(self, x):
# If x is a dict, convert it to list format.
if isinstance(x, (OrderedDict, dict)):
return [(k, v) for k, v in x.items()]
# Make sure x is a list or tuple.
if not isinstance(x, (list, tuple)):
msg = (
"Attempted to set options for '{}' widget with an object of type `{}`\n"
"Expected a list, tuple or OrderdDict."
)
raise ValueError(msg.format(self.__class__.__name__, type(x)))
# If x is an ordinary list, use the option values as names.
for y in x:
if not isinstance(y, (list, tuple)) or len(y) < 2:
return [(i, i) for i in x]
# Value is already in the correct format.
return x
def _options_changed(self, name, old, new):
"""Handles when the options tuple has been changed.
Setting options implies setting option labels from the keys of the dict.
"""
if self.options_lock.acquire(False):
try:
self.options = new
options = self._make_options(new)
self._options_dict = {i[0]: i[1] for i in options}
self._options_labels = [i[0] for i in options]
self._options_values = [i[1] for i in options]
self._value_in_options()
finally:
self.options_lock.release()
def _value_in_options(self):
# ensure that the chosen value is one of the choices
if self._options_values:
if self.value not in self._options_values:
self.value = next(iter(self._options_values))
def _options_readonly_changed(self, change):
if not self.options_lock.locked():
raise TraitError("`.%s` is a read-only trait. Use the `.options` tuple instead." % change['name'])
def _value_changed(self, name, old, new):
"""Called when value has been changed"""
if self.value_lock.acquire(False):
try:
# Reverse dictionary lookup for the value name
for k, v in self._options_dict.items():
if self.equals(new, v):
# set the selected value name
self.selected_label = k
return
# undo the change, and raise KeyError
self.value = old
raise KeyError(new)
finally:
self.value_lock.release()
def _selected_label_changed(self, name, old, new):
"""Called when the value name has been changed (typically by the frontend)."""
if self.value_lock.acquire(False):
try:
self.value = self._options_dict[new]
finally:
self.value_lock.release()
class EmergencyNode:
def __init__(self):
# subscribe to the laser data
rospy.Subscriber("/scan", LaserScan, self.emergency_callback)
# subscribe to the move commands and execute them as long as not in an emergency
rospy.Subscriber("/racecar/move_commands",AckermannDriveStamped, self.move_callback)
# advertise that we'll publish on the racecar/emergency_stop topic for notifications
self.emergency_pub = rospy.Publisher("racecar/emergency_stop", Bool, queue_size = 1)
# advertise that we'll publish on the cmd_vel topic for ackermann_mux_cmd
self.cmd_vel_pub = rospy.Publisher("vesc/ackermann_cmd_mux/input/safety", AckermannDriveStamped, queue_size=10)
# global variables
self.safety_distance = 0.5
self.emergency_time_delta = rospy.Duration.from_sec(0.5)
self.emergency_lock = Lock()
self.last_move_command = None
# stop command
msg = AckermannDriveStamped()
msg.header.stamp = rospy.Time.now()
msg.drive.steering_angle = 0.0
msg.drive.speed = 0.0
self.stop_command = msg
# reverse command
msg = AckermannDriveStamped()
msg.header.stamp = rospy.Time.now()
msg.drive.steering_angle = 0.0
msg.drive.speed = -1.0
self.reverse_command = msg
# See if obstacle is within cone of +/-30 degrees
def emergency_callback(self, LaserScan_msg):
if self.emergency_lock.acquire(False):
# Laser index 0:1080, using 420:660 for +/- 30 degrees
cone = list(LaserScan_msg.ranges[420:660])
print min(cone)
if min(cone) < self.safety_distance:
dangerIndicies = [(cone.index(i) + 420) for i in cone if i <= self.safety_distance]
#There might be dirt on the laser, so check if over threshold size
if len(dangerIndicies) > 5:
#publish emergency and stop the car for time delta
self.emergency_pub.publish(1)
self.cmd_vel_pub.publish(self.reverse_command)
rospy.Timer(self.emergency_time_delta,self.emergency_over,1)
else:
self.emergency_pub.publish(0)
self.emergency_lock.release()
else:
self.emergency_lock.release()
# release the lock and allow move commands because out of danger
def emergency_over(self,event):
self.emergency_pub.publish(0)
if not (self.last_move_command == None):
self.cmd_vel_pub.publish(self.last_move_command)
else:
self.cmd_vel_pub.publish(self.stop_command)
self.emergency_lock.release()
# Continue to publish move commands unless in an emergency
def move_callback(self, move_command):
if self.emergency_lock.locked():
self.last_move_command = move_command
else:
self.cmd_vel_pub.publish(move_command)
class BaseFoobnixControls():
def __init__(self):
self.vk_service = VKService(FC().access_token, FC().user_id)
self.count_errors = 0
self.is_scrobbled = False
self.start_time = None
self.chache_text = None
self.play_lock = Lock()
def check_for_media(self, args):
dirs = []
files = []
for arg in args:
if os.path.isdir(arg):
dirs.append(arg)
elif os.path.isfile(arg) and get_file_extension(arg) in FC().all_support_formats:
files.append(arg)
if dirs:
self.on_add_folders(dirs)
gobject.idle_add(self.play_first_file_in_playlist)
elif files:
self.on_add_files(files)
gobject.idle_add(self.play_first_file_in_playlist)
def love_this_tracks(self, beans=None):
if not beans:
return
map(self.lastfm_service.love, beans)
def show_google_results(self, query):
beans = []
beans.append(FModel('"%s" not found trying Google search' % query))
g_results = google_search_results(query)
for line in g_results:
beans.append(FModel(line).add_is_file(True))
if not g_results:
beans.append(FModel('Google not found %s' % query))
return beans
def get_active_bean(self):
tree = self.notetabs.get_current_tree()
if tree:
return tree.get_selected_or_current_bean()
def play_selected_song(self):
current = self.get_active_bean()
tree = self.notetabs.get_current_tree()
if not current:
try:
current = tree.get_bean_under_pointer_icon()
except AttributeError:
return
if not current:
return None
logging.debug("play current bean is %s" % str(current.text))
if current and current.is_file:
tree.set_play_icon_to_bean(current)
"""play radio, do not check VK"""
if current.type and current.type == FTYPE_RADIO:
self.play(current)
return
if current.path and current.path.startswith("http://"):
if not self.check_path(current.path):
res = self.net_wrapper.execute(self.vk_service.find_one_track, current.get_display_name())
if not res:
return
path = res.path
if path:
current.path = path
"""play song"""
self.play(current)
def check_path(self, path):
if path:
if not path.startswith("http://"):
if os.path.exists(path):
return True
else:
try:
"""Timeout not compatible with python 2.5"""
#u = urlopen(bean.path, timeout = 7) #@UnusedVariable
u = urlopen(path) #@UnusedVariable
if not vars().has_key("u"):
return False
return True
except:
return False
return False
def save_beans_to(self, beans):
return None
def on_chage_player_state(self, state, bean):
logging.debug("bean state %s" % (state))
if not FC().system_icons_dinamic:
return None
if state == STATE_STOP:
self.trayicon.set_image_from_path(FC().stop_icon_entry)
elif state == STATE_PAUSE:
self.trayicon.set_image_from_path(FC().pause_icon_entry)
elif state == STATE_PLAY:
self.trayicon.set_image_from_path(FC().play_icon_entry)
if bean and bean.type:
logging.debug("bean state and type %s %s" % (state, bean.type))
if bean.type == FTYPE_RADIO:
return self.trayicon.set_image_from_path(FC().radio_icon_entry)
def on_add_folders(self, paths=None):
if not paths:
paths = directory_chooser_dialog(_("Choose folders to open"), FC().last_dir)
if paths:
def task():
path = paths[0]
list = path.split("/")
FC().last_dir = path[:path.rfind("/")]
name = list[len(list) - 1]
parent = FModel(name)
self.append_to_new_notebook(name, [])
all_beans = []
all_beans.append(parent)
for bean in get_all_music_by_paths(paths, self):
if not bean.is_file:
bean.parent(parent).add_is_file(False)
all_beans.append(bean)
if all_beans:
self.append_to_current_notebook(all_beans)
else:
self.append([self.SearchCriteriaBeen(_("Nothing found to play in the folder(s)") + paths[0])])
self.in_thread.run_with_progressbar(task)
def on_add_files(self, paths=None, tab_name=None):
if not paths:
paths = file_chooser_dialog(_("Choose file to open"), FC().last_dir)
copy_paths = copy.deepcopy(paths)
for i, path in enumerate(copy_paths):
if path.lower().endswith(".m3u") or path.lower().endswith(".m3u8"):
paths[i:i + 1] = m3u_reader(path)
if len(copy_paths) == 1:
ext = os.path.splitext(path)[1]
tab_name = os.path.basename(path)[:-len(ext)]
break
if paths:
if paths[0]:
if isinstance(paths[0], list):
path = paths[0][0]
else:
path = paths[0]
else:
if isinstance(path, list):
path = paths[1][0]
else:
path = paths[1]
if path:
list_path = path.split("/")
name = list_path[len(list_path) - 2]
if not tab_name:
tab_name = os.path.split(os.path.dirname(path))[1]
FC().last_dir = path[:path.rfind("/")]
self.append_to_new_notebook(tab_name, [])
parent = FModel(name)
self.append_to_current_notebook([parent])
else:
self.append_to_new_notebook(tab_name, [])
parent = FModel(tab_name)
self.append_to_current_notebook([parent])
beans = []
for path in paths:
text = None
if isinstance(path, list):
text = path[1]
path = path[0]
bean = FModel(path, path).add_is_file(True)
else:
bean = FModel(path, path).parent(parent).add_is_file(True)
if text:
bean.text = text
beans.append(bean)
if not beans:
self.append_to_current_notebook([FModel(_("Nothing found to play in the file(s)") + paths[0])])
else:
self.append_to_current_notebook(beans)
def set_playlist_tree(self):
self.notetabs.set_playlist_tree()
def set_playlist_plain(self):
self.notetabs.set_playlist_plain()
def load_music_tree(self):
self.perspective.hide_add_button()
if not FCache().cache_music_tree_beans[0] and len(FCache().cache_music_tree_beans) == 1:
self.perspective.show_add_button()
self.tree.is_empty = True
if FCache().tab_names[0]:
self.tabhelper.label.set_label(FCache().tab_names[0] + " ")
else:
tabs = len(FCache().cache_music_tree_beans)
self.tree.simple_append_all(FCache().cache_music_tree_beans[tabs - 1])
self.tabhelper.label.set_label(FCache().tab_names[tabs - 1] + " ")
for tab in xrange(tabs - 2, -1, -1):
tree = NavigationTreeControl(self)
tree.simple_append_all(FCache().cache_music_tree_beans[tab])
self.tabhelper._append_tab(FCache().tab_names[tab], navig_tree=tree)
if not FCache().cache_music_tree_beans[tab]:
tree.is_empty = True
self.perspective.show_add_button()
logging.info("Tree loaded from cache")
if FC().update_tree_on_start:
def cycle():
for n in xrange(len(FCache().music_paths)):
tab_child = self.tabhelper.get_nth_page(n)
tree = tab_child.get_child()
self.update_music_tree(tree, n)
gobject.idle_add(cycle)
def update_music_tree(self, tree=None, number_of_page=0):
if not tree:
tree = self.tree
logging.info("Update music tree" + str(FCache().music_paths[number_of_page]))
tree.clear_tree()
FCache().cache_music_tree_beans[number_of_page] = []
all = []
all = get_all_music_by_paths(FCache().music_paths[number_of_page], self)
for bean in all:
FCache().cache_music_tree_beans[number_of_page].append(bean)
try:
self.perspective.hide_add_button()
except AttributeError:
logging.warn("Object perspective not exists yet")
if not all:
tree.is_empty = True
try:
self.perspective.show_add_button()
except AttributeError:
logging.warn("Object perspective not exists yet")
all.append(FModel(_("Music not found in folder(s):")))
for path in FCache().music_paths[number_of_page]:
all.append(FModel(path).add_is_file(True))
else: tree.is_empty = False
tree.append_all(all)
tree.ext_width = tree.ext_column.get_width()
def set_visible_video_panel(self, flag):
FC().is_view_video_panel = flag
if flag:
self.movie_window.show()
else:
self.movie_window.hide()
def volume_up(self):
self.volume.volume_up()
def volume_down(self):
self.volume.volume_down()
def mute(self):
self.volume.mute()
def hide(self):
self.main_window.hide()
def show_hide(self):
self.main_window.show_hide()
def show(self):
self.main_window.show()
def play_pause(self):
if self.media_engine.get_state() == STATE_PLAY:
self.media_engine.state_pause()
else:
self.media_engine.state_play()
def seek_up(self):
self.media_engine.seek_up()
def seek_down(self):
self.media_engine.seek_down()
def windows_visibility(self):
visible = self.main_window.get_property('visible')
if visible:
self.main_window.hide()
else:
self.main_window.show()
def state_play(self, remember_position=False, under_pointer_icon=False):
if self.media_engine.get_state() == STATE_PAUSE and not remember_position:
self.media_engine.state_play()
self.statusbar.set_text(self.media_engine.bean.info)
elif under_pointer_icon:
tree = self.notetabs.get_current_tree()
bean = tree.get_bean_under_pointer_icon()
self.play(bean)
else:
self.play_selected_song()
def show_preferences(self):
self.preferences.show()
def state_pause(self):
self.media_engine.state_pause()
def state_stop(self, remember_position=False):
self.record.hide()
self.media_engine.state_stop(remember_position)
if not remember_position:
self.statusbar.set_text(_("Stopped"))
self.seek_bar.clear()
def state_play_pause(self):
self.media_engine.state_play_pause()
bean = self.media_engine.bean
if self.media_engine.get_state() == STATE_PLAY:
self.statusbar.set_text(bean.info)
else:
self.statusbar.set_text(_("Paused | ") + str(bean.info))
def state_is_playing(self):
return self.media_engine.get_state() == STATE_PLAY
def fill_bean_from_vk(self, bean):
vk = self.vk_service.find_one_track(bean.get_display_name())
if vk:
bean.path = vk.path
bean.time = vk.time
return True
else:
return False
def play(self, bean):
if not bean or not bean.is_file:
return
self.play_lock.acquire()
if bean.type == FTYPE_RADIO:
self.record.show()
else:
self.record.hide()
def task():
self.seek_bar.clear()
self.statusbar.set_text(bean.info)
self.trayicon.set_text(bean.text)
self.movie_window.set_text(bean.text)
self.main_window.set_title(bean.text)
gobject.idle_add(task)
thread.start_new_thread(self._one_thread_play, (bean,))
#self._play(bean)
def _one_thread_play(self,bean):
try:
self._play(bean)
finally:
if self.play_lock.locked():
self.play_lock.release()
def _play(self, bean):
self.count_errors = 0
if not bean.path:
bean.path = get_bean_posible_paths(bean)
if bean.path and bean.type != FTYPE_RADIO and bean.path.startswith("http"):
if not url_utils.is_exists(bean.path):
bean.path = None
if not bean.path:
if not self.fill_bean_from_vk(bean):
def post_task():
self._play(bean)
if self.vk_service.is_show_authorization(post_task):
return None
if self.count_errors < 4:
time.sleep(0.5)
self.count_errors += 1
if self.play_lock.locked():
self.play_lock.release()
self.next()
if bean.path:
if not os.path.exists(bean.path):
if bean.iso_path and os.path.exists(bean.iso_path):
logging.info("Try to remount " + bean.iso_path)
mount_tmp_iso(bean.iso_path)
elif not bean.path.startswith("http"):
logging.error("File " + bean.path + " not found")
elif os.path.isdir(bean.path):
return None
self.media_engine.play(bean)
self.is_scrobbled = False
self.start_time = False
if not get_file_extension(bean.path) in FC().video_formats:
if bean.type != FTYPE_RADIO:
self.update_info_panel(bean)
self.set_visible_video_panel(False)
def notify_playing(self, pos_sec, dur_sec, bean, sec):
self.seek_bar.update_seek_status(pos_sec, dur_sec)
sec = int(sec)
if sec > 10 and sec % 11 == 0:
self.net_wrapper.execute(self.lastfm_service.report_now_playing, bean)
if not self.start_time:
self.start_time = str(int(time.time()))
if not self.is_scrobbled:
if sec > dur_sec / 2 or sec > 60:
self.is_scrobbled = True
self.net_wrapper.execute(self.lastfm_service.report_scrobbled, bean, self.start_time, dur_sec)
"""download music"""
if FC().automatic_online_save:
self.dm.append_task(bean)
def notify_title(self, text):
logging.debug("Notify title" + text)
self.statusbar.set_text(text)
text = normalize_text(text)
self.seek_bar.set_text(text)
t_bean = FModel(text).add_type(FTYPE_RADIO).create_from_text(text)
self.update_info_panel(t_bean)
if FC().enable_radio_scrobbler:
start_time = str(int(time.time()))
self.net_wrapper.execute(self.lastfm_service.report_now_playing, t_bean)
if " - " in text and self.chache_text != text:
text = self.chache_text
self.net_wrapper.execute(self.lastfm_service.report_scrobbled, t_bean, start_time, 200)
def notify_error(self, msg):
logging.error("notify error " + msg)
self.seek_bar.set_text(msg)
self.info_panel.clear()
def notify_eos(self):
self.next()
def player_seek(self, percent):
self.media_engine.seek(percent)
def player_volume(self, percent):
self.media_engine.volume(percent)
def search_vk_page_tracks(self, vk_ulr):
logging.debug("Search vk_service page tracks")
results = self.vk_service.find_tracks_by_url(vk_ulr)
all = []
p_bean = FModel(vk_ulr).add_font("bold")
all.append(p_bean)
for i, bean in enumerate(results):
bean.tracknumber = i + 1
bean.parent(p_bean).add_is_file(True)
all.append(bean)
self.notetabs.append_tab(vk_ulr, all)
def search_all_videos(self, query):
def inline():
results = self.vk_service.find_videos_by_query(query)
all = []
p_bean = FModel(query).add_font("bold")
all.append(p_bean)
for i, bean in enumerate(results):
bean.tracknumber = i + 1
bean.parent(p_bean).add_is_file(True)
all.append(bean)
if not results:
all = self.show_google_results(query)
self.notetabs.append_tab(query, all)
self.in_thread.run_with_progressbar(inline)
def search_all_tracks(self, query):
def inline():
results = self.vk_service.find_tracks_by_query(query)
if not results:
results = []
all = []
p_bean = FModel(query).add_font("bold")
all.append(p_bean)
for i, bean in enumerate(results):
bean.tracknumber = i + 1
bean.parent(p_bean).add_is_file(True)
all.append(bean)
if not results:
all = self.show_google_results(query)
self.notetabs.append_tab(query, all)
self.in_thread.run_with_progressbar(inline, no_thread=True)
def search_top_tracks(self, query):
def inline(query):
results = self.lastfm_service.search_top_tracks(query)
if not results:
results = []
all = []
parent_bean = FModel(query)
all.append(parent_bean)
for i, bean in enumerate(results):
bean.tracknumber = i + 1
bean.parent(parent_bean).add_is_file(True)
all.append(bean)
if not results:
all = self.show_google_results(query)
self.notetabs.append_tab(query, all)
self.in_thread.run_with_progressbar(inline, query)
def search_top_albums(self, query):
def inline(query):
results = self.lastfm_service.search_top_albums(query)
if not results:
results = []
self.notetabs.append_tab(query, None)
albums_already_inserted = []
for album in results[:15]:
all = []
if (album.album.lower() in albums_already_inserted):
continue
album.is_file = False
tracks = self.lastfm_service.search_album_tracks(album.artist, album.album)
for i, track in enumerate(tracks):
track.tracknumber = i + 1
track.album = album.album
track.parent(album).add_is_file(True)
all.append(track)
if (len(all) > 0):
all = [album] + all
albums_already_inserted.append(album.album.lower())
self.notetabs.append_all(all)
if not results:
all = self.show_google_results(query)
self.notetabs.append_all(all)
self.in_thread.run_with_progressbar(inline, query)
def search_top_similar(self, query):
def inline(query):
results = self.lastfm_service.search_top_similar_artist(query)
if not results:
results = []
self.notetabs.append_tab(query, None)
for artist in results[:15]:
all = []
artist.is_file = False
all.append(artist)
tracks = self.lastfm_service.search_top_tracks(artist.artist)
for i, track in enumerate(tracks):
track.tracknumber = i + 1
track.parent(artist).add_is_file(True)
all.append(track)
self.notetabs.append_all(all)
if not results:
all = self.show_google_results(query)
#inline(query)
self.in_thread.run_with_progressbar(inline, query)
def search_top_tags(self, query):
def inline(query):
results = self.lastfm_service.search_top_tags(query)
if not results:
logging.debug("tag result not found")
results = []
self.notetabs.append_tab(query, None)
for tag in results[:15]:
all = []
tag.is_file = False
all.append(tag)
tracks = self.lastfm_service.search_top_tag_tracks(tag.text)
for i, track in enumerate(tracks):
track.tracknumber = i + 1
track.parent(tag).add_is_file(True)
all.append(track)
self.notetabs.append_all(all)
if not results:
all = self.show_google_results(query)
self.notetabs.append_all(all)
#inline(query)
self.in_thread.run_with_progressbar(inline, query)
def update_info_panel(self, bean):
self.info_panel.update(bean)
def append_to_new_notebook(self, text, beans, optimization=False):
self.notetabs._append_tab(text, beans, None, optimization)
def append_to_current_notebook(self, beans):
self.notetabs.append_all(beans)
def next(self):
bean = self.notetabs.next()
if not bean:
return
gap = FC().gap_secs
time.sleep(gap)
logging.debug("play current bean is %s" % str(bean.text))
if bean.path:
if os.path.isdir(bean.path):
return None
if bean.path.startswith("http://"):
if not self.check_path(bean.path):
path = self.net_wrapper.execute(self.vk_service.find_one_track, bean.get_display_name()).path
if path:
bean.path = path
self.play(bean)
def prev(self):
bean = self.notetabs.prev()
if not bean:
return
if bean.path:
if os.path.isdir(bean.path):
return None
if bean.path.startswith("http://"):
if not self.check_path(bean.path):
path = self.net_wrapper.execute(self.vk_service.find_one_track, bean.get_display_name()).path
if path:
bean.path = path
self.play(bean)
def filter_by_folder(self, value):
tree = self.tabhelper.get_current_tree()
tree.filter_by_folder(value)
self.radio.filter_by_folder(value)
self.virtual.filter_by_folder(value)
def filter_by_file(self, value):
tree = self.tabhelper.get_current_tree()
tree.filter_by_file(value)
self.radio.filter_by_file(value)
self.virtual.filter_by_file(value)
self.vk_integration.filter_by_folder(query=value, expand=False)
def quit(self, *a):
self.state_stop()
self.main_window.hide()
self.trayicon.hide()
logging.info("Controls - Quit")
self.notetabs.on_quit()
self.virtual.on_quit()
self.info_panel.on_quit()
self.radio.on_quit()
self.my_radio.on_quit()
FC().save()
gtk.main_quit()
def check_version(self):
uuid = FCBase().uuid
current_version = FOOBNIX_VERSION
system = "not_set"
try:
import platform
system = platform.system()
except:
pass
try:
from socket import gethostname
f = urllib2.urlopen("http://www.foobnix.com/version?uuid=" + uuid + "&host=" + gethostname() + "&version=" + current_version + "&platform=" + system)
#f = urllib2.urlopen("http://localhost:8080/version?uuid=" + uuid + "&host=" + gethostname() + "&v=" + current_version)
except Exception, e:
logging.error("Check version error: " + str(e))
return None
new_version_line = f.read()
logging.info("version " + current_version + "|" + new_version_line + "|" + str(uuid))
f.close()
if FC().check_new_version and compare_versions(current_version, new_version_line) == 1:
info_dialog_with_link_and_donate(new_version_line)
class LocationController(object):
IDEALBACKLOG = 5
MAXPARALLELCHANNELS = 10
MAXTRANSPERCHANNEL = 250
ZOMBIETIMEOUT = 900
def __init__(self, monitor, host, port, protocol = 'http', debug = 0):
self.monitor = monitor
self.host = host
self.port = port
self.protocol = protocol
self.debug = debug
self.free_channels = []
self.active_channels = Dictionary()
self.inactive_channels = Dictionary()
self.parallel_channels = Counter()
self.pending_transactions = []
self.inflight_transactions = Dictionary()
self.state_lock = Lock()
self.debugout('Instantiated', 2)
def is_transaction_pending(self,sid):
for tran in self.pending_transactions:
if(tran.sid == sid):
return(True)
return(False)
def recruit_and_send(self):
self.state_lock.acquire()
self.debugout('Pending Transactions in recruit and send are :%d' %(len(self.pending_transactions)))
try:
openchannel = self._recruit_channel()
finally:
self.state_lock.release()
if openchannel:
self.debugout('Initiating a transaction by recruit and send' )
self.initiate_next_transaction()
else:
self.debugout('Recruit channel returns None. Unable to init a transaction by recruit and send')
def add_transaction(self, transaction):
self.state_lock.acquire()
self.debugout('Pending Transactions are :%d' %(len(self.pending_transactions)))
try:
self.debugout('[%s] Adding transaction', 3)
transaction.set_manager(self)
openchannel = self._recruit_channel()
self.pending_transactions.append(transaction)
finally:
self.state_lock.release()
if openchannel:
self.initiate_next_transaction()
else:
self.debugout('Recruit channel returns None. Pending transactions =%d' %(len(self.pending_transactions)))
def initiate_next_transaction(self):
self.state_lock.acquire()
try:
channel = self._activate_channel()
if channel is not None:
transaction = self._activate_transaction()
finally:
self.state_lock.release()
if channel is None:
message = 'Activate channel returned None to initiate next.'
message += '\n\t\t-- %r\n'
self.debugout(message, 1, self)
else:
transaction.set_channel(channel)
message = '[%s] Initiating transaction.'
self.debugout(message, 3)
if channel.response_count() > self.MAXTRANSPERCHANNEL:
message = '[%s] Configuring transaction to close channel:'
message += '\n\t%r\n\t%r\n'
self.debugout(message, 1, transaction, channel)
transaction.close_when_done()
transaction.initiate()
def handle_completed_transaction(self, transaction):
self.debugout('[%s] Handling completed transaction', 3)
self.state_lock.acquire()
try:
self._deactivate_transaction(transaction)
self._deactivate_channel(transaction.channel)
if self.pending_transactions:
initiatenext = (self.inactive_channels or
self._recruit_channel())
else:
initiatenext = False
self._free_channel(transaction.channel)
finally:
self.state_lock.release()
if self.debug > 1:
message = '[%s] completed: ' % self
message += '%s' % transaction.stats()[1:-1]
self.msglog(message, msglog.types.DB, False)
if initiatenext:
self.initiate_next_transaction()
def handle_failed_transaction(self, transaction):
try:
self.state_lock.acquire()
try:
# Changes for CSCtg33093 (b.Avoid getting the channel number, if channel is None)
if(transaction.channel != None ):
transaction.channel.accepting_requests(False)
finally:
self.state_lock.release()
message = 'Handling failed transaction %r' % transaction
self.msglog(message, msglog.types.WARN)
finally:
self.handle_completed_transaction(transaction)
def _activate_transaction(self):
assert self.state_lock.locked()
transaction = self.pending_transactions.pop(0)
tid = transaction.transaction_number
self.inflight_transactions[tid] = transaction
return transaction
def _deactivate_transaction(self, transaction):
assert self.state_lock.locked()
tid = transaction.transaction_number
try:
self.inflight_transactions.pop(tid)
except KeyError:
self.debugout('Transaction %r not in active list', 1, transaction)
return False
else:
return True
def _recruit_channel(self):
assert self.state_lock.locked()
if not self.free_channels:
self._manage_channels()
channel = None
if self.free_channels:
channel = self.free_channels.pop()
cid = channel.channel_number
self.inactive_channels[cid] = channel
return channel
def _free_channel(self, channel):
assert self.state_lock.locked()
cid = channel.channel_number
if self.inactive_channels.has_key(cid):
self.inactive_channels.pop(cid)
if channel.accepting_requests():
self.free_channels.append(channel)
def _activate_channel(self):
assert self.state_lock.locked()
if not self.inactive_channels:
channel = self._recruit_channel()
else:
cid, channel = self.inactive_channels.popitem()
if channel is not None:
self.active_channels[channel.channel_number] = channel
return channel
def _deactivate_channel(self, channel):
assert self.state_lock.locked()
# Changes for CSCtg33093 (b.Avoid getting the channel number, if channel is None)
if(channel == None):
return(False)
cid = channel.channel_number
try:
self.inactive_channels[cid] = self.active_channels.pop(cid)
except KeyError:
self.debugout('Channel %r not in active list', 1, channel)
return False
else:
if not channel.accepting_requests():
self.inactive_channels.pop(cid)
self.parallel_channels.decrement()
if not channel.is_closed():
channel.close()
self.debugout('Channel %r decommissioned.', 1, channel)
return True
def _manage_channels(self):
assert self.state_lock.locked()
for cid, channel in self.active_channels.items():
message = 'Location channel management removing %r from active.'
if not channel.accepting_requests():
self.debugout(message % channel, 1)
self.active_channels.pop(cid)
self.parallel_channels.decrement()
maxchannels = self.MAXPARALLELCHANNELS
numpending = len(self.pending_transactions)
numchannels = self.parallel_channels.value
if numchannels == 0:
createchannel = True
elif numchannels >= maxchannels:
createchannel = False
elif (numpending / numchannels) > self.IDEALBACKLOG:
createchannel = True
else:
createchannel = False
if createchannel:
channel = Channel(self.monitor, self.debug)
channel.setup_connection(self.host, self.port, self.protocol)
self.free_channels.append(channel)
self.parallel_channels.increment()
return
def msglog(self, message, mtype = msglog.types.INFO, autoprefix = False):
if autoprefix:
message = '%s %s' % (self, message)
msglog.log('broadway', mtype, message)
def debugout(self, message, debuglevel = 1, *args):
if debuglevel <= self.debug:
if self not in args:
if message.count('%') == len(args):
message = '%s ' + message
args = (self,) + args
return self.msglog(message % args, msglog.types.DB)
def __repr__(self):
status = ['Controller']
status.append('[%s:%d]' % (self.host, self.port))
channels = self.parallel_channels.value
active = len(self.active_channels)
inactive = len(self.inactive_channels)
free = len(self.free_channels)
pending = len(self.pending_transactions)
inflight = len(self.inflight_transactions)
channelmsg = '%d Channels (%dA %dI %dF)'
status.append(channelmsg % (channels, active, inactive, free))
status.append('Transactions (%dA %dP)' % (inflight, pending))
return '<%s>' % (' '.join(status))
def __str__(self):
classname = self.__class__.__name__
channels = self.parallel_channels.value
status = '(%d => %s:%d)' % (channels, self.host, self.port)
return '%s %s' % (classname, status)
class NaptListener(object):
def __init__(self, bindaddr):
super().__init__()
self.lock = Lock()
self.bindaddr = bindaddr
self.sockets = {} # Dictionary<int, Socket>
self.status = NaptListenerStatus.Stopped
self.port = 0
self.accepted = Event2('NaptListenerEventArgs')
def __str__(self):
return 'NaptListener{ %s }' %', '.join([
'bindaddr=%s' % str(self.bindaddr)])
# public
def add_port(self, port):
with self.lock:
if port in self.sockets:
raise Exception() # ArgumentException(nameof(port))
self.listen(port)
# private
def listen(self, port):
# todo ここでBindしないでStart時のBind,Stop時にCloseする
so = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
bindaddr= '0.0.0.0' if self.bindaddr == 'any' else self.bindaddr
endpoint= (bindaddr, port)
self.port = port
print(' %s' % str(endpoint))
so.bind(endpoint)
so.listen(10)
SocketPoller.get_instance().register(so, SocketPollFlag.ReadError, self.polling_callback)
# public
def remove_port(self, port):
with self.lock:
so = self.sockets[port]
SocketPoller.get_instance().unregister(so)
self.sockets.pop(port)
so.close()
# protected virtual
def on_accepted(self, e): # NaptListenerEventArgs
self.accepted(self, e)
# private
def polling_callback_thread(self, so, flag):
if 0 != (flag & SocketPollFlag.Read):
self.do_recv(so)
if 0 != (flag & SocketPollFlag.Write):
self.do_send(so)
if 0 != (flag & SocketPollFlag.Error):
self.do_error(so)
def polling_callback(self, so, flag):
t = threading.Thread(target=self.polling_callback_thread, args=([so, flag]))
t.start()
t.join(60.0)
if t.is_alive() == True :
# Timeout
so.close()
# protected override
def do_recv(self, so):
Utils.expects_type(socket.socket, so, 'so')
try:
so.settimeout(3.0)
so_accepted, remote= so.accept();
so_accepted.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 0, 0))
e = NaptListenerEventArgs(so_accepted, so, self.port);
self.on_accepted(e)
except Exception as ex:
print('do_recv: Exception', flush=True)
Utils.print_exception(ex)
print('', flush=True)
try:
key = [k for k, v in self.sockets.items() if v == so][0]
self.remove_port(key)
except Exception as ex:
pass
#raise ex
# protected override
def do_send(self, so):
Utils.expects_type(socket.socket, so, 'so')
# protected override
def do_error(self, so):
Utils.expects_type(socket.socket, so, 'so')
# todo error
# protected override
def update_select_sockets(self):
Utils.assertion(self.lock.locked(), 'need lock')
self.read_sockets.extend(self.sockets.values())
self.error_sockets.extend(self.sockets.values())
class RelevantCrawler(APIV3Crawler):
"""Relevant list crawler bases on YouTube API V3 crawler."""
def __init__(self):
APIV3Crawler.__init__(self)
self._mutex_update = Lock()
self._setup_logger('metadatacrawler')
# Reading developer_key from developer.key file from conf, roll over when exceed quota
try:
with open('conf/developer.key', 'r') as keys:
developer_keys = keys.readlines()
self.set_keys(developer_keys)
self.set_key_index(0)
except:
self.logger.error('**> The keys file developer.key does not exist! It should be in the conf folder.')
sys.exit(1)
self._youtube = None
self._update_youtube()
def _update_youtube(self):
self._youtube = discovery.build(self._api_service, self._api_version, developerKey=self._keys[self._key_index], cache_discovery=False)
def _search_relevant_list(self, vid, pageToken=None):
"""Finds the relevant list about a specifies videoId from youtube and returns video id associated with it."""
# start_time = time.time()
if self._key_index >= len(self._keys):
self.logger.error('Key index out of range, exit.')
os._exit(0)
else:
current_key_index = self._key_index
# Call the search().list method to retrieve results matching the specified video term.
try:
search_response = self._youtube.search().list(part="snippet", relatedToVideoId=vid, type='video', maxResults=50, pageToken=pageToken).execute()
except errors.HttpError as error:
if error.resp.status == 403:
if (not self._mutex_update.locked()) and current_key_index == self._key_index:
self._mutex_update.acquire()
self.logger.error('The request cannot be completed because quota exceeded.')
self.logger.error('Current developer key index {0}.'.format(current_key_index))
if self._key_index < len(self._keys):
self.set_key_index(self._key_index+1)
self.logger.error('Updated developer key index {0}.'.format(self._key_index))
self._mutex_update.release()
time.sleep(random.random())
self._update_youtube()
return self._search_relevant_list(vid, pageToken=pageToken)
else:
raise error
# Check to get empty responses handled properly
try:
if len(search_response['items']) == 0:
# self.logger.debug('Request for {0} request number was empty.'.format(vid))
return
else:
res = []
for video in search_response['items']:
rid = video['id']['videoId']
res.append(str(rid))
if 'nextPageToken' in search_response:
next_pagetoken = search_response['nextPageToken']
# recursively request next page
if next_pagetoken:
next_page = self._search_relevant_list(vid, pageToken=next_pagetoken)
if next_page:
res.extend(next_page)
except Exception as e:
self.logger.error('Request for {0} failed with error {1} while processing.'.format(vid, str(e)))
return
return res
def _youtube_search_with_exponential_backoff(self, vid):
""" Implements Exponential backoff on youtube search."""
for i in xrange(0, 5):
try:
return self._search_relevant_list(vid)
except errors.HttpError as error:
if error.resp.status == 500 or error.resp.status == 503:
time.sleep((2 ** i) + random.random())
else:
self.logger.error('Request for {0} failed with error code {1} at invocation.'.format(vid, error.resp.status))
break
self.logger.error('Request for {0} request has an error and never succeeded.'.format(vid))
def start(self, input_file, output_dir):
self.logger.warning('**> Outputting result to files...')
# define the two queues: one for working jobs, one for results.
to_process = Queue()
to_write = Queue()
# action executed by the worked threads
# they input lines from the working queue, process them, obtain the JSON object and put it into writing queue
def worker():
while True:
# make sure that our workers don't die on us :)
try:
vid = to_process.get()
# process the file only if it was not already done
relevant_list = self._youtube_search_with_exponential_backoff(vid)
if relevant_list:
relevant_list.insert(0, vid)
to_write.put(','.join(relevant_list))
except Exception as exc:
self.logger.error('[Input Queue] Error in writing: {0}.'.format(str(exc)))
to_process.task_done()
def writer():
"""Function to take values from the output queue and write it to a file
"""
output_path = '{0}/relevant_list.txt'
video_metadata = open(output_path.format(output_dir), 'w')
while True:
try:
jobj = to_write.get()
# check for file termination object
if jobj == 0:
self.logger.warning('**> Termination object received and wait for termination...')
video_metadata.close()
elif jobj is not None:
video_metadata.write('{0}\n'.format(jobj))
except Exception as e:
self.logger.error('[Output Queue] Error in writing: {0}.'.format(str(e)))
# in any case, mark the current item as done
to_write.task_done()
# start the working threads - 10 of them
for i in range(self._num_threads):
t = Thread(target=worker)
t.daemon = True
t.start()
# start the writer thread
w = Thread(target=writer)
w.daemon = True
w.start()
# all is good, start the work
# opening vid file and reading the video id file to retrieve data
with open(input_file, mode='r') as datafile:
initial_time = time.time()
for line in datafile:
vid = line.rstrip().split()[0]
to_process.put(vid)
# wait for jobs to be done
to_process.join()
# give the termination object and wait for termination
to_write.put(0)
to_write.join()
self.logger.warning('**> Total time for requests {0:.4f} secs.'.format(time.time() - initial_time))
sys.exit(0)
class ROSImageSubscriber(Thread):
'''
Generic tool for ros images
'''
def __init__(self,topics,queue_size=1,use_compression=True,loop_rate = 1.0):
Thread.__init__(self)
self.node_name = 'img_subscriber'
try:
rospy.init_node(self.node_name, anonymous=True)
except: pass
self.caller_id = rospy.get_name()
self.caller_id = self.caller_id[len(self.node_name)+1:]
if not isinstance(topics, list):
topics = [topics]
self.topics = topics
self.pid = os.getpid()
#self.daemon = False
if(use_compression):
self.start_compression_threads()
print(str(self)+' subscribing to : '+str(self.topics))
self.mutex = [Lock()]*len(topics)
self.has_received_first = [False]*len(topics)
self.should_register_mouse = [False]*len(topics)
self.mouse_function = [self.wtf]*len(topics)
self.bridge = CvBridge()
self.images=[]
self._stop = threading.Event()
self.lock_ = Lock()
self.loopy = rospy.Rate(loop_rate)
if len(self.topics) == 1:
rospy.Subscriber(self.topics[0], Image,self.callback,queue_size=queue_size)
self.images.append(np.array([]))
else:
sub=[]
for topic in self.topics:
sub.append(message_filters.Subscriber(topic, Image))
self.images.append(np.array([]))
ts = message_filters.TimeSynchronizer(sub, queue_size)
ts.registerCallback(self.callback)
def lock(self):
self.lock_.acquire()
def release(self):
self.lock_.release()
def locked(self):
return self.lock_.locked()
def start_compression_threads(self):
self.comp=[]
for i in xrange(0,len(self.topics)):
isdepth = self.topics[i].find('/depth') >=0
if not isdepth:
self.comp.append(CompressTopic(self.topics[i],self.caller_id,compress_mode='compressed'))
else:
self.comp.append(CompressTopic(self.topics[i],self.caller_id,compress_mode='compressedDepth'))
for i in xrange(0,len(self.topics)):
self.topics[i] = self.comp[i].topic_out
self.comp[i].start()
def register_mouse_callback(self,function):
for i in xrange(len(self.topics)):
self.mouse_function[i] = function
self.should_register_mouse[i] = True
def mouse_callback_spin_once(self):
for i in xrange(len(self.topics)):
if self.should_register_mouse[i]:
window = self.topics[i]+str(i)
function = self.mouse_function[i]
print 'Registering',function,'for window name',window
cv2.setMouseCallback(window,function)
self.should_register_mouse[i] = False
def get_window_name(self):
if len(self.topics)==1:
window = self.topics[0]+str(0)
else:
window = []
for i in xrange(len(self.topics)):
window.append(self.topics[i]+str(i))
return window
def stop(self):
self._stop.set()
print self,' stopped'
def wtf(self):
for i in xrange(0,len(self.topics)):
if self.has_received_first[i]:
print 'I have received at least 1 image from : ',self.topics[i]
else:
print "I'm still waiting for : ",self.topics[i]
def run(self):
rospy.spin()
try:
while not rospy.is_shutdown():
self.loopy.sleep()
except KeyboardInterrupt:
self.stop()
for i in xrange(len(self.images)):
self.mutex[i].acquire()
#cv2.destroyAllWindows()#(self.topics[i]+str(i))
cv2.destroyWindow(self.topics[i]+str(i))
self.mutex[i].release()
def callback(self,*msg):
if self.locked():
#print "locked"
return
for i in xrange(len(self.topics)):
try:
enc = msg[i].encoding
if enc=='rgb8':
# Strange kinect exception
enc='bgr8'
if enc=='32FC1':
enc='passthrough'
if not self.has_received_first[i]:
self.has_received_first[i] = True
self.mutex[i].acquire()
try:
self.images[i] = self.bridge.imgmsg_to_cv2(msg[i],enc)
except Exception,e:
print "Warning, using old cv brdige : ",e
self.images[i] = np.array(self.bridge.imgmsg_to_cv(msg[i],enc))
self.mutex[i].release()
except CvBridgeError, e:
print e
class GstGain:
def __init__(self, uri, ref_level):
self.__lock = Lock()
self.uri = uri
self.ref_level = ref_level
self.result = (False, 0, 0, uri)
self.gain_pipe = None
# Create a pipeline with a fake audio output and get the gain levels
def gain(self):
pipe = 'uridecodebin uri="{0}" ! audioconvert ! rganalysis \
reference-level={1} ! fakesink'.format(self.uri, self.ref_level)
self.gain_pipe = Gst.parse_launch(pipe)
gain_bus = self.gain_pipe.get_bus()
gain_bus.add_signal_watch()
gain_bus.connect("message", self._on_message)
logging.info('REPLY-GAIN:: started ' + str(self.uri))
self.gain_pipe.set_state(Gst.State.PLAYING)
# Block here until EOS
self.__lock.acquire(False)
self.__lock.acquire()
# Reset the pipe
self.gain_pipe = None
# Return the computation result
return self.result
def stop(self):
if self.gain_pipe is not None:
self.gain_pipe.send_event(Gst.Event.new_eos())
def _on_message(self, bus, message):
try:
if message.type == Gst.MessageType.EOS and self.__lock.locked():
self.__lock.release()
elif message.type == Gst.MessageType.TAG:
tags = message.parse_tag()
tag = tags.get_double(Gst.TAG_TRACK_GAIN)
peak = tags.get_double(Gst.TAG_TRACK_PEAK)
if tag[0] and peak[0]:
self.gain_pipe.set_state(Gst.State.NULL)
self.result = (True, tag[1], peak[1], self.uri)
if self.__lock.locked():
self.__lock.release()
elif message.type == Gst.MessageType.ERROR:
logging.debug('REPLY-GAIN:: ' + str(message.parse_error()))
self.gain_pipe.set_state(Gst.State.NULL)
if self.__lock.locked():
self.__lock.release()
except Exception:
if self.__lock.locked():
self.__lock.release()
