def _post_connect(self):
"Greeting stuff"
init_event = Event()
error = [None] # so that err_cb can bind error[0]. just how it is.
# callbacks
def ok_cb(id, capabilities):
self._id = id
self._server_capabilities = capabilities
init_event.set()
def err_cb(err):
error[0] = err
init_event.set()
listener = HelloHandler(ok_cb, err_cb)
self.add_listener(listener)
self.send(HelloHandler.build(self._client_capabilities))
logger.debug('starting main loop')
self.start()
# we expect server's hello message
init_event.wait()
# received hello message or an error happened
self.remove_listener(listener)
if error[0]:
raise error[0]
#if ':base:1.0' not in self.server_capabilities:
# raise MissingCapabilityError(':base:1.0')
logger.info('initialized: session-id=%s | server_capabilities=%s' %
(self._id, self._server_capabilities))
class HeartbeatFuture(object):
def __init__(self, connection, owner):
self._exception = None
self._event = Event()
self.connection = connection
self.owner = owner
log.debug("Sending options message heartbeat on idle connection (%s) %s", id(connection), connection.host)
with connection.lock:
if connection.in_flight < connection.max_request_id:
connection.in_flight += 1
connection.send_msg(OptionsMessage(), connection.get_request_id(), self._options_callback)
else:
self._exception = Exception("Failed to send heartbeat because connection 'in_flight' exceeds threshold")
self._event.set()
def wait(self, timeout):
self._event.wait(timeout)
if self._event.is_set():
if self._exception:
raise self._exception
else:
raise OperationTimedOut()
def _options_callback(self, response):
if not isinstance(response, SupportedMessage):
if isinstance(response, ConnectionException):
self._exception = response
else:
self._exception = ConnectionException(
"Received unexpected response to OptionsMessage: %s" % (response,)
)
log.debug("Received options response on connection (%s) from %s", id(self.connection), self.connection.host)
self._event.set()
class ReceiveNotification(object):
def __init__(self, address, pstream):
self.received = Event()
self.requester = Requester(self.received, pstream, address, False, "hci1")
self.connect()
self.requester.write_by_handle(0x3C, str(bytearray([0xff, 0xff])))
self.requester.write_by_handle(0x3E, str(bytearray([0x64])))
data = self.requester.read_by_handle(0x3C)[0]
for d in data:
print(hex(ord(d)), end=' ')
print("")
self.requester.write_by_handle(0x3A, str(bytearray([0x1, 0x0])))
self.wait_notification()
def connect(self):
print("Connecting...", end=' ')
sys.stdout.flush()
self.requester.connect()
print("OK!")
def wait_notification(self):
print("\nThis is a bit tricky. You need to make your device to send\n"
"some notification. I'll wait...")
self.received.wait()
class TimerWithResume(object):
def __init__(self, status_subject, refresh_interval):
self.status_subject = status_subject
self.abort = Event()
self.refresh_interval = refresh_interval
def perform(self):
while not self.abort.isSet():
self.status_subject.build_status()
self.abort.wait(self.refresh_interval)
def stop(self):
self.abort.set()
def start(self):
self.thread = Thread(target=self.perform)
self.thread.daemon = True
self.thread.start()
def resume(self):
self.thread.join()
self.abort.clear()
self.start()
def set_refresh_interval(self, new_interval):
self.refresh_interval = new_interval
class ResponseWaiter(object):
def __init__(self, connection, num_responses):
self.connection = connection
self.pending = num_responses
self.error = None
self.responses = [None] * num_responses
self.event = Event()
def got_response(self, response, index):
with self.connection.lock:
self.connection.in_flight -= 1
if isinstance(response, Exception):
self.error = response
self.event.set()
else:
self.responses[index] = response
self.pending -= 1
if not self.pending:
self.event.set()
def deliver(self, timeout=None):
self.event.wait(timeout)
if self.error:
raise self.error
elif not self.event.is_set():
raise OperationTimedOut()
else:
return self.responses
class TestWatchMixin(object):
"""Testing the watch command is hard."""
def watch_loop(self):
# Hooked into the loop of the ``watch`` command.
# Allows stopping the thread.
self.has_looped.set()
time.sleep(0.01)
if getattr(self, 'stopped', False):
return True
def start_watching(self):
"""Run the watch command in a thread."""
self.has_looped = Event()
t = Thread(target=self.cmd_env.watch, kwargs={'loop': self.watch_loop})
t.daemon = True # In case something goes wrong with stopping, this
# will allow the test process to be end nonetheless.
t.start()
self.t = t
# Wait for first iteration, which will initialize the mtimes. Only
# after this will ``watch`` be able to detect changes.
self.has_looped.wait(1)
def stop_watching(self):
"""Stop the watch command thread."""
assert self.t.isAlive() # If it has already ended, something is wrong
self.stopped = True
self.t.join(1)
def __enter__(self):
self.start_watching()
def __exit__(self, exc_type, exc_val, exc_tb):
self.stop_watching()
class Actor:
def __init__(self):
self._mailbox = Queue()
def send(self, msg):
self._mailbox.put(msg)
def recv(self):
msg = self._mailbox.get()
if msg is ActorExit:
raise ActorExit()
return msg
def start(self):
self._terminated = Event()
t = Thread(target=self._bootstrap)
t.daemon = True
t.start()
def _bootstrap(self):
try:
self.run()
except ActorExit:
pass
finally:
self._terminated.set()
def join(self):
self._terminated.wait()
def run(self):
while True:
msg = self.recv()
class StoppableQThread(QtCore.QThread):
""" Base class for QThreads which require the ability
to be stopped by a thread-safe method call
"""
def __init__(self, parent=None):
self._should_stop = Event()
self._should_stop.clear()
super(StoppableQThread, self).__init__(parent)
def join(self, timeout=0):
""" Joins the current thread and forces it to stop after
the timeout if necessary
:param timeout: Timeout duration in seconds
"""
self._should_stop.wait(timeout)
if not self.should_stop():
self.stop()
super(StoppableQThread, self).wait()
def stop(self):
self._should_stop.set()
def should_stop(self):
return self._should_stop.is_set()
def __repr__(self):
return "<%s(should_stop=%s)>" % (self.__class__.__name__, self.should_stop())
def waitForBackend(self, devid):
frontpath = self.frontendPath(devid)
# lookup a phantom
phantomPath = xstransact.Read(frontpath, 'phantom_vbd')
if phantomPath is not None:
log.debug("Waiting for %s's phantom %s.", devid, phantomPath)
statusPath = phantomPath + '/' + HOTPLUG_STATUS_NODE
ev = Event()
result = { 'status': Timeout }
xswatch(statusPath, hotplugStatusCallback, ev, result)
ev.wait(DEVICE_CREATE_TIMEOUT)
err = xstransact.Read(statusPath, HOTPLUG_ERROR_NODE)
if result['status'] != 'Connected':
return (result['status'], err)
backpath = xstransact.Read(frontpath, "backend")
if backpath:
statusPath = backpath + '/' + HOTPLUG_STATUS_NODE
ev = Event()
result = { 'status': Timeout }
xswatch(statusPath, hotplugStatusCallback, ev, result)
ev.wait(DEVICE_CREATE_TIMEOUT)
err = xstransact.Read(backpath, HOTPLUG_ERROR_NODE)
return (result['status'], err)
else:
return (Missing, None)
def _callback(self, request):
""" This method is called by the ROS framework when a Service request
has arrived.
Each call runs in a separate thread and has to block until a
response is present, because the return value of this method is
used as response to the request.
"""
msgID = uuid4().hex
event = Event()
with self._pendingLock:
self._pending[msgID] = event
self._reactor.callFromThread(self.received, request._buff, msgID)
# Block execution here until the event is set, i.e. a response has
# arrived
event.wait()
with self._pendingLock:
response = self._pending.pop(msgID, None)
if not isinstance(response, Message):
# TODO: Change exception?
raise rospy.ROSInterruptException('Interrupted.')
return response
class DeferredResponse( object ):
"""
A deferred that resolves to a response from TSServer.
"""
def __init__( self, timeout = RESPONSE_TIMEOUT_SECONDS ):
self._event = Event()
self._message = None
self._timeout = timeout
def resolve( self, message ):
self._message = message
self._event.set()
def result( self ):
self._event.wait( timeout = self._timeout )
if not self._event.isSet():
raise RuntimeError( 'Response Timeout' )
message = self._message
if not message[ 'success' ]:
raise RuntimeError( message[ 'message' ] )
if 'body' in message:
return self._message[ 'body' ]
def sync(loop, func, *args, **kwargs):
""" Run coroutine in loop running in separate thread """
if not loop._running:
try:
return loop.run_sync(lambda: func(*args, **kwargs))
except RuntimeError: # loop already running
pass
from threading import Event
e = Event()
result = [None]
error = [False]
traceback = [False]
@gen.coroutine
def f():
try:
result[0] = yield gen.maybe_future(func(*args, **kwargs))
except Exception as exc:
logger.exception(exc)
result[0] = exc
error[0] = exc
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback[0] = exc_traceback
finally:
e.set()
a = loop.add_callback(f)
while not e.is_set():
e.wait(1000000)
if error[0]:
six.reraise(type(error[0]), error[0], traceback[0])
else:
return result[0]
class ResponseEvent:
"""Event which is fired when the response is returned for a request.
For each request sent this event is created.
An application can wait for the event to create a blocking request.
"""
def __init__(self):
self.__evt = Event()
def waiting(self):
return not self.__evt.isSet()
def waitForResponse(self, timeOut=None):
"""blocks until the response arrived or timeout is reached."""
self.__evt.wait(timeOut)
if self.waiting():
raise Timeout()
else:
if self.response["error"]:
raise Exception(self.response["error"])
else:
return self.response["result"]
def handleResponse(self, resp):
self.response = resp
self.__evt.set()
class Queue(list):
def __init__(self):
super(Queue, self).__init__()
self._lock = Lock()
self._fill = Event()
def put(self, obj):
with self._lock:
self.append(obj)
self._fill.set()
def get(self, block=True):
with self._lock:
if len(self) == 0:
self._fill.clear()
if not self._fill.isSet():
if block:
self._fill.wait()
else:
return None
with self._lock:
return self.pop(0)
def delete(self, index):
if 0 <= index < len(self):
with self._lock:
del self[index]
def remove(self, element):
if element in self:
with self._lock:
del self[self.index(element)]
class TestInterruptibleDecorator(TestCase):
def setUp(self):
self.quit_condition = False
self.thread_started = Event()
@interruptible
def never_ending(self, cancellation_point):
self.thread_started.set()
while True:
time.sleep(0.01)
cancellation_point()
def test_interruptible_decorator(self):
""" Tests for the @interruptible decorator. """
thread = Thread(target=self.never_ending, args=(
lambda: _cancellation_point(lambda: self.quit_condition),))
thread.start()
# Wait until thread comes to live
self.thread_started.wait()
# Ask to it to quit within 20ms
self.quit_condition = True
time.sleep(0.02)
# Thread is finished
self.assertFalse(thread.is_alive())
class TempBackup(CopyDir):
def __init__(self, cwd, prefix='backup_', timeout=5):
temp = mkdtemp(prefix=prefix)
super().__init__(cwd, temp)
self.suspend = Event()
self.progress = Event()
self.progress.clear()
self.suspend.set()
self.timeout = timeout
def resume(self):
self.suspend.set()
def join(self, timeout=None):
self.resume()
super().join(timeout)
def wait(self):
while not self.progress.wait(self.timeout):
print('waiting for backup to continue..')
self.progress.clear()
def run(self):
super().run()
self.progress.set()
self.suspend.clear()
while not self.suspend.wait(self.timeout):
print('waiting for ok to delete tempbakup')
shutil.rmtree(self.target)
print('deleted {}'.format(self.target))
def TriggerEventWait(
self,
suffix,
payload=None,
prefix="Main",
source=eg
):
event = EventGhostEvent(suffix, payload, prefix, source)
if event.source in self.filters:
for filterFunc in self.filters[event.source]:
if filterFunc(event) is True:
return event
executed = Event()
def Execute():
try:
event.Execute()
finally:
executed.set()
def Transfer():
ActionThreadCall(Execute)
event.SetShouldEnd()
self.AppendAction(Transfer)
executed.wait(5.0)
if not executed.isSet():
eg.PrintWarningNotice(
"timeout TriggerEventWait\n", traceback.format_stack()
)
return event
def test_no_dupes(self, mock_prep, mock_proc, mock_get):
e = Event()
blk = MultiQueryREST(e)
mock_get.return_value = Response()
mock_get.return_value.status_code = 200
mock_proc.return_value = [
Signal({'_id': 1}),
Signal({'_id': 2})
], False
self.configure_block(blk, {
"polling_interval": {
"seconds": 0.5
},
"retry_interval": {
"seconds": 1
},
"queries": [
"foobar",
"bazqux"
]
})
blk.start()
e.wait(2)
self.assert_num_signals_notified(2, blk)
blk.stop()
def test_poll(self, mock_prep, mock_proc, mock_get):
e = Event()
blk = RESTBlock(e)
mock_get.return_value = Response()
mock_get.return_value.status_code = 200
mock_proc.return_value = [None, None]
self.configure_block(blk, {
"polling_interval": {
"seconds": 1
},
"retry_interval": {
"seconds": 1
},
"queries": [
"foobar"
]
})
blk.start()
e.wait(2)
mock_prep.assert_called_once_with(False)
self.assertEqual(mock_get.call_count, 1)
self.assertEqual(mock_proc.call_count, 1)
blk.stop()
class ThreadedRunner(Runnable):
def __init__(self, runnable):
self._runnable = runnable
self._notifier = Event()
self._result = None
self._error = None
self._traceback = None
self._thread = None
def run(self):
try:
self._result = self._runnable()
except:
self._error, self._traceback = sys.exc_info()[1:]
self._notifier.set()
__call__ = run
def run_in_thread(self, timeout):
self._thread = Thread(self, name=TIMEOUT_THREAD_NAME)
self._thread.setDaemon(True)
self._thread.start()
self._notifier.wait(timeout)
return self._notifier.isSet()
def get_result(self):
if self._error:
raise self._error, None, self._traceback
return self._result
def stop_thread(self):
self._thread.stop()
def test_paging(self, mock_prep, mock_proc, mock_get):
e = Event()
blk = RESTBlock(e)
mock_get.return_value = Response()
mock_get.return_value.status_code = 200
mock_proc.side_effect = [([None, None], True),
([None, None], True),
([None, None], False)]
self.configure_block(blk, {
"polling_interval": {
"seconds": 1
},
"retry_interval": {
"seconds": 1
},
"queries": [
"foobar"
]
})
blk.start()
e.wait(2)
self.assertEqual(blk.page_num, 3)
blk.stop()
def test_term_thread(self):
"""ctx.term should not crash active threads (#139)"""
ctx = self.Context()
evt = Event()
evt.clear()
def block():
s = ctx.socket(zmq.REP)
s.bind_to_random_port('tcp://127.0.0.1')
evt.set()
try:
s.recv()
except zmq.ZMQError as e:
self.assertEqual(e.errno, zmq.ETERM)
return
finally:
s.close()
self.fail("recv should have been interrupted with ETERM")
t = Thread(target=block)
t.start()
evt.wait(1)
self.assertTrue(evt.is_set(), "sync event never fired")
time.sleep(0.01)
ctx.term()
t.join(timeout=1)
self.assertFalse(t.is_alive(), "term should have interrupted s.recv()")
def join(self,server,channel,nick=None,port=6667):
channel = channel.lower()
self.connect(server,port,nick)
status = self.get_status(server)
connected_servers = set(status['servers'])
if not server in connected_servers:
print("Not connected...")
if channel in set(status['servers'][server]['channels']):
print("Joined already.")
return
else:
print("Joining",channel)
e = Event()
def channel_joined(addr,event):
if event['kind']=='irc':
if event['command']=='JOIN':
if event['trailing'].lower()==channel:
status['servers'][server]['channels'].append(channel)
self.events.unlisten(server,channel_joined)
e.set()
self.events.listen(server,channel_joined)
self.events.broadcast({
'event':'irc.command:'+server,
'command':"JOIN",
"arguments":[channel]
})
e.wait()
class WaitableTimer(Timer):
def __init__(self, timeout, callback):
Timer.__init__(self, timeout, callback)
self.callback = callback
self.event = Event()
self.final_exception = None
def finish(self, time_now):
try:
finished = Timer.finish(self, time_now)
if finished:
self.event.set()
return True
return False
except Exception as e:
self.final_exception = e
self.event.set()
return True
def wait(self, timeout=None):
self.event.wait(timeout)
if self.final_exception:
raise self.final_exception
class RepeatTimer(Thread):
def __init__(self, interval, function, stop_on_exception=True, args=[], kwargs={}):
Thread.__init__(self)
self.interval = interval
self.function = function
self.args = args
self.kwargs = kwargs
self.stop_on_exception = stop_on_exception
self.finished = Event()
def stop(self):
self.done = True
self.finished.set()
def run(self):
self.done = False
while True:
self.finished.wait(self.interval)
if self.done:
self.finished.clear()
break
if not self.finished.is_set():
try:
self.function(*self.args, **self.kwargs)
except Exception, e:
if self.stop_on_exception:
self.finished.clear()
raise
# XXX Not strictly for auth'ing, think of something better
log.exception('Login Fail')
self.finished.clear()
class CheckForUpdates(Thread):
INTERVAL = 24*60*60 # seconds
daemon = True
def __init__(self, parent):
Thread.__init__(self)
self.shutdown_event = Event()
self.signal = Signal(parent)
def run(self):
while not self.shutdown_event.is_set():
calibre_update_version = NO_CALIBRE_UPDATE
plugins_update_found = 0
try:
version = get_newest_version()
if version[:2] > numeric_version[:2]:
calibre_update_version = version
except Exception as e:
prints('Failed to check for calibre update:', as_unicode(e))
try:
update_plugins = get_plugin_updates_available(raise_error=True)
if update_plugins is not None:
plugins_update_found = len(update_plugins)
except Exception as e:
prints('Failed to check for plugin update:', as_unicode(e))
if calibre_update_version != NO_CALIBRE_UPDATE or plugins_update_found > 0:
self.signal.update_found.emit(calibre_update_version, plugins_update_found)
self.shutdown_event.wait(self.INTERVAL)
def shutdown(self):
self.shutdown_event.set()
class Worker(Thread):
def __init__(self):
Thread.__init__(self)
self.trigger = Event()
self.uri = None
self.sonos = None
self.daemon = True
self.lock = Lock()
def start_url(self, a_group, uri):
with self.lock:
self.sonos = a_group
self.uri = uri
self.trigger.set()
def run(self):
while True:
self.trigger.wait()
if self.sonos is None:
# take this as a sign to shut down gracefully
break
with self.lock:
uri = self.uri
self.uri = None
self.trigger.clear()
result = self.sonos.play_uri( uri)
print result
class InterruptableEvent(object):
"""Event class for for Python v2.7 which behaves more like Python v3.2
threading.Event and allows signals to interrupt waits"""
def __init__(self):
self.__event = Event()
def is_set(self):
return self.__event.is_set()
def set(self):
self.__event.set()
def clear(self):
self.__event.clear()
def wait(self, timeout=None):
# infinite
if timeout is None:
# event with timeout is interruptable
while not self.__event.wait(60):
pass
return True
# finite
else:
# underlying Event will perform timeout argument validation
return self.__event.wait(timeout)
class TestInterruptible(unittest.TestCase):
""" Tests for interrupting cooperative threads """
def test_interruptible_decorator(self):
""" Tests for the @interruptible decorator. """
self.quit_condition = False
cancellation_point = lambda: _cancellation_point(
lambda: self.quit_condition)
self.thread_started = Event()
@interruptible
def never_ending(cancellation_point):
self.thread_started.set()
while True:
time.sleep(0.1)
cancellation_point()
thread = Thread(target=never_ending, args=(cancellation_point, ))
thread.start()
self.thread_started.wait()
self.quit_condition = True
countdown = 10
while thread.is_alive() and countdown > 0:
time.sleep(0.1)
countdown -= 1
self.assertFalse(thread.is_alive())
class WaitApp (object):
def __init__ (self):
# self.event = event
self.ev = Event( )
self.loop = GLib.MainLoop( )
self.expired = False
def handle_emitted (self, status):
print "emitted", status, self
def handle_event (self):
print "event", self.loop
self.loop.quit( )
def until (self, event, timeout=None):
self.event = event
self.event.Do.connect(self.handle_event)
self.event.Emit.connect(self.handle_emitted)
self.background = Thread(target=self.pending, args=(timeout, self.loop.quit))
self.background.daemon = True
self.background.start( )
self.loop.run( )
def pending (self, timeout, quit):
print "starting background, waiting for ", timeout
self.ev.wait(timeout)
quit( )
self.expired = True
print "Failed to find event within", timeout
class Session:
VERSION = __version__
APP_VERSION = "Pyrogram \U0001f525 {}".format(VERSION)
DEVICE_MODEL = "{} {}".format(
platform.python_implementation(),
platform.python_version()
)
SYSTEM_VERSION = "{} {}".format(
platform.system(),
platform.release()
)
INITIAL_SALT = 0x616e67656c696361
NET_WORKERS = 1
WAIT_TIMEOUT = 10
MAX_RETRIES = 5
ACKS_THRESHOLD = 8
PING_INTERVAL = 5
notice_displayed = False
BAD_MSG_DESCRIPTION = {
16: "[16] msg_id too low, the client time has to be synchronized",
17: "[17] msg_id too high, the client time has to be synchronized",
18: "[18] incorrect two lower order msg_id bits, the server expects client message msg_id to be divisible by 4",
19: "[19] container msg_id is the same as msg_id of a previously received message",
20: "[20] message too old, it cannot be verified by the server",
32: "[32] msg_seqno too low",
33: "[33] msg_seqno too high",
34: "[34] an even msg_seqno expected, but odd received",
35: "[35] odd msg_seqno expected, but even received",
48: "[48] incorrect server salt",
64: "[64] invalid container"
}
def __init__(self,
dc_id: int,
test_mode: bool,
proxy: type,
auth_key: bytes,
api_id: str,
is_cdn: bool = False,
client: pyrogram = None):
if not Session.notice_displayed:
print("Pyrogram v{}, {}".format(__version__, __copyright__))
print("Licensed under the terms of the " + __license__, end="\n\n")
Session.notice_displayed = True
self.connection = Connection(DataCenter(dc_id, test_mode), proxy)
self.api_id = api_id
self.is_cdn = is_cdn
self.client = client
self.auth_key = auth_key
self.auth_key_id = sha1(auth_key).digest()[-8:]
self.session_id = Long(MsgId())
self.msg_factory = MsgFactory()
self.current_salt = None
self.pending_acks = set()
self.recv_queue = Queue()
self.results = {}
self.ping_thread = None
self.ping_thread_event = Event()
self.next_salt_thread = None
self.next_salt_thread_event = Event()
self.is_connected = Event()
def start(self):
terms = None
while True:
try:
self.connection.connect()
for i in range(self.NET_WORKERS):
Thread(target=self.net_worker, name="NetWorker#{}".format(i + 1)).start()
Thread(target=self.recv, name="RecvThread").start()
self.current_salt = FutureSalt(0, 0, self.INITIAL_SALT)
self.current_salt = FutureSalt(0, 0, self._send(functions.Ping(0)).new_server_salt)
self.current_salt = self._send(functions.GetFutureSalts(1)).salts[0]
self.next_salt_thread = Thread(target=self.next_salt, name="NextSaltThread")
self.next_salt_thread.start()
if not self.is_cdn:
terms = self._send(
functions.InvokeWithLayer(
layer,
functions.InitConnection(
self.api_id,
self.DEVICE_MODEL,
self.SYSTEM_VERSION,
self.APP_VERSION,
"en", "", "en",
functions.help.GetTermsOfService(),
)
)
).text
self.ping_thread = Thread(target=self.ping, name="PingThread")
self.ping_thread.start()
log.info("Connection inited: Layer {}".format(layer))
except (OSError, TimeoutError, Error):
self.stop()
else:
break
self.is_connected.set()
log.debug("Session started")
return terms
def stop(self):
self.is_connected.clear()
self.ping_thread_event.set()
self.next_salt_thread_event.set()
if self.ping_thread is not None:
self.ping_thread.join()
if self.next_salt_thread is not None:
self.next_salt_thread.join()
self.ping_thread_event.clear()
self.next_salt_thread_event.clear()
self.connection.close()
for i in range(self.NET_WORKERS):
self.recv_queue.put(None)
log.debug("Session stopped")
def restart(self):
self.stop()
self.start()
def pack(self, message: Message):
data = Long(self.current_salt.salt) + self.session_id + message.write()
padding = urandom(-(len(data) + 12) % 16 + 12)
# 88 = 88 + 0 (outgoing message)
msg_key_large = sha256(self.auth_key[88: 88 + 32] + data + padding).digest()
msg_key = msg_key_large[8:24]
aes_key, aes_iv = KDF(self.auth_key, msg_key, True)
return self.auth_key_id + msg_key + AES.ige_encrypt(data + padding, aes_key, aes_iv)
def unpack(self, b: BytesIO) -> Message:
assert b.read(8) == self.auth_key_id, b.getvalue()
msg_key = b.read(16)
aes_key, aes_iv = KDF(self.auth_key, msg_key, False)
data = BytesIO(AES.ige_decrypt(b.read(), aes_key, aes_iv))
data.read(8)
# https://core.telegram.org/mtproto/security_guidelines#checking-session-id
assert data.read(8) == self.session_id
message = Message.read(data)
# https://core.telegram.org/mtproto/security_guidelines#checking-sha256-hash-value-of-msg-key
# https://core.telegram.org/mtproto/security_guidelines#checking-message-length
# 96 = 88 + 8 (incoming message)
assert msg_key == sha256(self.auth_key[96:96 + 32] + data.getvalue()).digest()[8:24]
# https://core.telegram.org/mtproto/security_guidelines#checking-msg-id
# TODO: check for lower msg_ids
assert message.msg_id % 2 != 0
return message
def net_worker(self):
name = threading.current_thread().name
log.debug("{} started".format(name))
while True:
packet = self.recv_queue.get()
if packet is None:
break
try:
self.unpack_dispatch_and_ack(packet)
except Exception as e:
log.error(e, exc_info=True)
log.debug("{} stopped".format(name))
def unpack_dispatch_and_ack(self, packet: bytes):
data = self.unpack(BytesIO(packet))
messages = (
data.body.messages
if isinstance(data.body, MsgContainer)
else [data]
)
log.debug(data)
for msg in messages:
if msg.seq_no % 2 != 0:
if msg.msg_id in self.pending_acks:
continue
else:
self.pending_acks.add(msg.msg_id)
if isinstance(msg.body, (types.MsgDetailedInfo, types.MsgNewDetailedInfo)):
self.pending_acks.add(msg.body.answer_msg_id)
continue
if isinstance(msg.body, types.NewSessionCreated):
continue
msg_id = None
if isinstance(msg.body, (types.BadMsgNotification, types.BadServerSalt)):
msg_id = msg.body.bad_msg_id
elif isinstance(msg.body, (core.FutureSalts, types.RpcResult)):
msg_id = msg.body.req_msg_id
elif isinstance(msg.body, types.Pong):
msg_id = msg.body.msg_id
else:
if self.client is not None:
self.client.updates_queue.put(msg.body)
if msg_id in self.results:
self.results[msg_id].value = getattr(msg.body, "result", msg.body)
self.results[msg_id].event.set()
if len(self.pending_acks) >= self.ACKS_THRESHOLD:
log.info("Send {} acks".format(len(self.pending_acks)))
try:
self._send(types.MsgsAck(list(self.pending_acks)), False)
except (OSError, TimeoutError):
pass
else:
self.pending_acks.clear()
def ping(self):
log.debug("PingThread started")
while True:
self.ping_thread_event.wait(self.PING_INTERVAL)
if self.ping_thread_event.is_set():
break
try:
self._send(functions.PingDelayDisconnect(0, self.PING_INTERVAL + 15), False)
except (OSError, TimeoutError):
pass
log.debug("PingThread stopped")
def next_salt(self):
log.debug("NextSaltThread started")
while True:
now = datetime.now()
# Seconds to wait until middle-overlap, which is
# 15 minutes before/after the current/next salt end/start time
dt = (self.current_salt.valid_until - now).total_seconds() - 900
log.debug("Current salt: {} | Next salt in {:.0f}m {:.0f}s ({})".format(
self.current_salt.salt,
dt // 60,
dt % 60,
now + timedelta(seconds=dt)
))
self.next_salt_thread_event.wait(dt)
if self.next_salt_thread_event.is_set():
break
try:
self.current_salt = self._send(functions.GetFutureSalts(1)).salts[0]
except (OSError, TimeoutError):
self.connection.close()
break
log.debug("NextSaltThread stopped")
def recv(self):
log.debug("RecvThread started")
while True:
packet = self.connection.recv()
if packet is None or len(packet) == 4:
if packet:
log.warning("Server sent \"{}\"".format(Int.read(BytesIO(packet))))
if self.is_connected.is_set():
Thread(target=self.restart, name="RestartThread").start()
break
self.recv_queue.put(packet)
log.debug("RecvThread stopped")
def _send(self, data: Object, wait_response: bool = True):
message = self.msg_factory(data)
msg_id = message.msg_id
if wait_response:
self.results[msg_id] = Result()
payload = self.pack(message)
try:
self.connection.send(payload)
except OSError as e:
self.results.pop(msg_id, None)
raise e
if wait_response:
self.results[msg_id].event.wait(self.WAIT_TIMEOUT)
result = self.results.pop(msg_id).value
if result is None:
raise TimeoutError
elif isinstance(result, types.RpcError):
Error.raise_it(result, type(data))
elif isinstance(result, types.BadMsgNotification):
raise Exception(self.BAD_MSG_DESCRIPTION.get(
result.error_code,
"Error code {}".format(result.error_code)
))
else:
return result
def send(self, data: Object):
for i in range(self.MAX_RETRIES):
self.is_connected.wait()
try:
return self._send(data)
except (OSError, TimeoutError):
log.warning("Retrying {}".format(type(data)))
continue
else:
return None
class KafkaStream(Thread):
def __init__(self, connection_info, advanced_info, topic_in, topic_out,
predictor, _type):
self.connection_info = connection_info
self.advanced_info = advanced_info
self.predictor = predictor
self.stream_in_name = topic_in
self.stream_out_name = topic_out
self.consumer = kafka.KafkaConsumer(
**self.connection_info, **self.advanced_info.get('consumer', {}))
self.consumer.subscribe(topics=[self.stream_in_name])
self.producer = kafka.KafkaProducer(
**self.connection_info, **self.advanced_info.get('producer', {}))
self.admin = kafka.KafkaAdminClient(**self.connection_info)
try:
self.topic = NewTopic(self.stream_out_name,
num_partitions=1,
replication_factor=1)
self.admin.create_topics([self.topic])
except kafka.errors.TopicAlreadyExistsError:
pass
self._type = _type
self.native_interface = NativeInterface()
self.format_flag = 'explain'
self.stop_event = Event()
self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
self.caches = {}
if self._type == 'timeseries':
super().__init__(target=KafkaStream.make_timeseries_predictions,
args=(self, ))
else:
super().__init__(target=KafkaStream.make_prediction, args=(self, ))
def _get_target(self):
return "pnew_case"
# pass
def _get_window_size(self):
return 10
# pass
def _get_gb(self):
return "state"
# pass
def _get_dt(self):
return "time"
# pass
def predict_ts(self, cache_name):
when_list = [x for x in self.caches[cache_name]]
for x in when_list:
if self.target not in x:
x['make_predictions'] = False
else:
x['make_predictions'] = True
result = self.native_interface.predict(self.predictor,
self.format_flag,
when_data=when_list)
log.error(f"TIMESERIES STREAM: got {result}")
for res in result:
in_json = json.dumps(res)
to_send = in_json.encode('utf-8')
log.error(f"sending {to_send}")
self.producer.send(self.stream_out_name, to_send)
self.caches[cache_name] = self.caches[cache_name][1:]
def make_prediction_from_cache(self, cache_name):
cache = self.caches[cache_name]
log.error("STREAM: in make_prediction_from_cache")
if len(cache) >= self.window:
log.error(
f"STREAM: make_prediction_from_cache - len(cache) = {len(cache)}"
)
self.predict_ts(cache_name)
def to_cache(self, record):
gb_val = record[self.gb]
cache_name = f"cache.{gb_val}"
if cache_name not in self.caches:
cache = []
self.caches[cache_name] = cache
log.error(f"STREAM: cache {cache_name} has been created")
self.make_prediction_from_cache(cache_name)
self.handle_record(cache_name, record)
self.make_prediction_from_cache(cache_name)
log.error("STREAM in cache: current iteration has done.")
def handle_record(self, cache_name, record):
log.error(f"STREAM: handling cache {cache_name} and {record} record.")
cache = self.caches[cache_name]
cache.append(record)
cache = self.sort_cache(cache)
self.caches[cache_name] = cache
def sort_cache(self, cache):
return sorted(cache, key=lambda x: x[self.dt])
def make_timeseries_predictions(self):
log.error("STREAM: running 'make_timeseries_predictions'")
predict_record = session.query(DBPredictor).filter_by(
company_id=self.company_id, name=self.predictor).first()
if predict_record is None:
log.error(
f"Error creating stream: requested predictor {self.predictor} is not exist"
)
return
self.target = self._get_target()
self.window = self._get_window_size()
self.gb = self._get_gb()
self.dt = self._get_dt()
while not self.stop_event.wait(0.5):
try:
msg_str = next(self.consumer)
when_data = json.loads(msg_str.value)
self.to_cache(when_data)
except StopIteration:
pass
log.error("Stopping stream..")
self.producer.close()
self.consumer.close()
session.close()
def make_prediction(self):
predict_record = session.query(DBPredictor).filter_by(
company_id=self.company_id, name=self.predictor).first()
if predict_record is None:
log.error(
f"Error creating stream: requested predictor {self.predictor} is not exist"
)
return
while not self.stop_event.wait(0.5):
try:
msg_str = next(self.consumer)
when_data = json.loads(msg_str.value)
result = self.native_interface.predict(self.predictor,
self.format_flag,
when_data=when_data)
log.error(f"STREAM: got {result}")
for res in result:
in_json = json.dumps({"prediction": res})
to_send = in_json.encode('utf-8')
log.error(f"sending {to_send}")
self.producer.send(self.stream_out_name, to_send)
except StopIteration:
pass
log.error("Stopping stream..")
self.producer.close()
self.consumer.close()
session.close()
class MultiGetPool(object):
"""
Encapsulates a pool of fetcher threads. These threads can be used
across many multi-get requests.
"""
def __init__(self, size=POOL_SIZE):
"""
:param size: the desired size of the worker pool
:type size: int
"""
self._inq = Queue()
self._size = size
self._started = Event()
self._stop = Event()
self._lock = Lock()
self._workers = []
def enq(self, task):
"""
Enqueues a fetch task to the pool of workers. This will raise
a RuntimeError if the pool is stopped or in the process of
stopping.
:param task: the Task object
:type task: Task
"""
if not self._stop.is_set():
self._inq.put(task)
else:
raise RuntimeError("Attempted to enqueue a fetch operation while "
"multi-get pool was shutdown!")
def start(self):
"""
Starts the worker threads if they are not already started.
This method is thread-safe and will be called automatically
when executing a MultiGet operation.
"""
# Check whether we are already started, skip if we are.
if not self._started.is_set():
# If we are not started, try to capture the lock.
if self._lock.acquire(False):
# If we got the lock, go ahead and start the worker
# threads, set the started flag, and release the lock.
for i in range(self._size):
name = "riak.client.multiget-worker-{0}".format(i)
worker = Thread(target=self._fetcher, name=name)
worker.daemon = True
worker.start()
self._workers.append(worker)
self._started.set()
self._lock.release()
else:
# We didn't get the lock, so someone else is already
# starting the worker threads. Wait until they have
# signaled that the threads are started.
self._started.wait()
def stop(self):
"""
Signals the worker threads to exit and waits on them.
"""
self._stop.set()
for worker in self._workers:
worker.join()
def stopped(self):
"""
Detects whether this pool has been stopped.
"""
return self._stop.is_set()
def __del__(self):
# Ensure that all work in the queue is processed before
# shutting down.
self.stop()
def _fetcher(self):
"""
The body of the multi-get worker. Loops until
:meth:`_should_quit` returns ``True``, taking tasks off the
input queue, fetching the object, and putting them on the
output queue.
"""
while not self._should_quit():
task = self._inq.get()
try:
btype = task.client.bucket_type(task.bucket_type)
obj = btype.bucket(task.bucket).get(task.key, **task.options)
task.outq.put(obj)
except KeyboardInterrupt:
raise
except Exception as err:
task.outq.put((task.bucket_type, task.bucket, task.key, err), )
finally:
self._inq.task_done()
def _should_quit(self):
"""
Worker threads should exit when the stop flag is set and the
input queue is empty. Once the stop flag is set, new enqueues
are disallowed, meaning that the workers can safely drain the
queue before exiting.
:rtype: bool
"""
return self.stopped() and self._inq.empty()
class RevPiSlave(Thread):
"""RevPi PLC-Server.
Diese Klasste stellt den RevPi PLC-Server zur verfuegung und akzeptiert
neue Verbindungen. Diese werden dann als RevPiSlaveDev abgebildet.
Ueber die angegebenen ACLs koennen Zugriffsbeschraenkungen vergeben werden.
"""
def __init__(self, ipacl, port=55234, bindip="", watchdog=True):
"""Instantiiert RevPiSlave-Klasse.
@param ipacl AclManager <class 'IpAclManager'>
@param port Listen Port fuer plc Slaveserver
@param bindip IP-Adresse an die der Dienst gebunden wird (leer=alle)
@param watchdog Trennen, wenn Verarbeitungszeit zu lang
"""
if not isinstance(ipacl, IpAclManager):
raise ValueError("parameter ipacl must be <class 'IpAclManager'>")
if not (isinstance(port, int) and 0 < port <= 65535):
raise ValueError(
"parameter port must be <class 'int'> and in range 1 - 65535")
if not isinstance(bindip, str):
raise ValueError("parameter bindip must be <class 'str'>")
super().__init__()
self.__ipacl = ipacl
self._bindip = bindip
self._evt_exit = Event()
self.exitcode = None
self._port = port
self.so = None
self._th_dev = []
self._watchdog = watchdog
self.zeroonerror = False
self.zeroonexit = False
def check_connectedacl(self):
"""Prueft bei neuen ACLs bestehende Verbindungen."""
for dev in self._th_dev:
ip, port = dev._addr
level = self.__ipacl.get_acllevel(ip)
if level < 0:
# Verbindung killen
proginit.logger.warning(
"client {0} not in acl - disconnect!".format(ip))
dev.stop()
elif level != dev._acl:
# ACL Level anpassen
proginit.logger.warning(
"change acl level from {0} to {1} on existing "
"connection {2}".format(level, dev._acl, ip))
dev._acl = level
def disconnect_all(self):
"""Close all device connection."""
# Alle Threads beenden
for th in self._th_dev:
th.stop()
def disconnect_replace_ios(self):
"""Close all device with loaded replace_ios file."""
# Alle Threads beenden die Replace_IOs emfpangen haben
for th in self._th_dev:
if th.got_replace_ios:
th.stop()
def newlogfile(self):
"""Konfiguriert die FileHandler auf neue Logdatei."""
pass
def run(self):
"""Startet Serverkomponente fuer die Annahme neuer Verbindungen."""
proginit.logger.debug("enter RevPiSlave.run()")
# Socket öffnen und konfigurieren bis Erfolg oder Ende
self.so = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.so.settimeout(2)
sock_bind_err = False
while not self._evt_exit.is_set():
try:
self.so.bind((self._bindip, self._port))
if sock_bind_err:
proginit.logger.warning(
"successful bind picontrolserver to socket "
"after error")
except Exception as e:
if not sock_bind_err:
sock_bind_err = True
proginit.logger.warning(
"can not bind picontrolserver to socket: {0} "
"- retrying".format(e))
self._evt_exit.wait(1)
else:
self.so.listen(32)
break
# Mit Socket arbeiten
while not self._evt_exit.is_set():
self.exitcode = -1
# Verbindung annehmen
try:
tup_sock = self.so.accept()
proginit.logger.info("accepted new connection for revpinetio")
except socket.timeout:
continue
except Exception:
if not self._evt_exit.is_set():
proginit.logger.exception("accept exception")
continue
# ACL prüfen
aclstatus = self.__ipacl.get_acllevel(tup_sock[1][0])
if aclstatus == -1:
tup_sock[0].close()
proginit.logger.warning(
"host ip '{0}' does not match revpiacl - disconnect"
"".format(tup_sock[1][0]))
else:
# Thread starten
th = RevPiSlaveDev(tup_sock, aclstatus, self._watchdog)
th.start()
self._th_dev.append(th)
# Liste von toten threads befreien
self._th_dev = [
th_check for th_check in self._th_dev if th_check.is_alive()
]
# Disconnect all clients and wait some time, because they are daemons
th_close_err = False
for th in self._th_dev: # type: RevPiSlaveDev
th.stop()
for th in self._th_dev: # type: RevPiSlaveDev
th.join(2.0)
if th.is_alive():
th_close_err = True
if th_close_err:
proginit.logger.warning(
"piControlServer could not disconnect all clients in timeout")
# Socket schließen
self.so.close()
self.so = None
self.exitcode = 0
proginit.logger.debug("leave RevPiSlave.run()")
def stop(self):
"""Beendet Slaveausfuehrung."""
proginit.logger.debug("enter RevPiSlave.stop()")
self._evt_exit.set()
if self.so is not None:
try:
self.so.shutdown(socket.SHUT_RDWR)
except Exception:
pass
proginit.logger.debug("leave RevPiSlave.stop()")
@property
def watchdog(self):
return self._watchdog
@watchdog.setter
def watchdog(self, value):
self._watchdog = value
for th in self._th_dev: # type: RevPiSlaveDev
th.watchdog = value
class DevWorker(object):
prefix = attr.ib(type=str, default="MANUAL:")
report_stdout = deferred_config('development.worker.log_stdout', True)
report_period = deferred_config('development.worker.report_period_sec',
30.,
transform=lambda x: float(max(x, 1.0)))
ping_period = deferred_config('development.worker.ping_period_sec',
30.,
transform=lambda x: float(max(x, 1.0)))
def __init__(self):
self._dev_stop_signal = None
self._thread = None
self._exit_event = Event()
self._task = None
self._support_ping = False
def ping(self, timestamp=None):
try:
if self._task:
self._task.send(tasks.PingRequest(self._task.id))
except Exception:
return False
return True
def register(self, task, stop_signal_support=None):
if self._thread:
return True
if (stop_signal_support is None
and TaskStopSignal.enabled) or stop_signal_support is True:
self._dev_stop_signal = TaskStopSignal(task=task)
self._support_ping = hasattr(tasks, 'PingRequest')
# if there is nothing to monitor, leave
if not self._support_ping and not self._dev_stop_signal:
return
self._task = task
self._exit_event.clear()
self._thread = Thread(target=self._daemon)
self._thread.daemon = True
self._thread.start()
return True
def _daemon(self):
last_ping = time()
while self._task is not None:
try:
if self._exit_event.wait(
min(float(self.ping_period),
float(self.report_period))):
return
# send ping request
if self._support_ping and (time() - last_ping) >= float(
self.ping_period):
self.ping()
last_ping = time()
if self._dev_stop_signal:
stop_reason = self._dev_stop_signal.test()
if stop_reason and self._task:
self._task._dev_mode_stop_task(stop_reason)
except Exception:
pass
def unregister(self):
self._dev_stop_signal = None
self._task = None
self._thread = None
self._exit_event.set()
return True
class MetadataBackup(Thread):
'''
Continuously backup changed metadata into OPF files
in the book directory. This class runs in its own
thread.
'''
def __init__(self, db, interval=2, scheduling_interval=0.1):
Thread.__init__(self)
self.daemon = True
self._db = weakref.ref(getattr(db, 'new_api', db))
self.stop_running = Event()
self.interval = interval
self.scheduling_interval = scheduling_interval
@property
def db(self):
ans = self._db()
if ans is None:
raise Abort()
return ans
def stop(self):
self.stop_running.set()
def wait(self, interval):
if self.stop_running.wait(interval):
raise Abort()
def run(self):
while not self.stop_running.is_set():
try:
self.wait(self.interval)
self.do_one()
except Abort:
break
def do_one(self):
try:
book_id = self.db.get_a_dirtied_book()
if book_id is None:
return
except Abort:
raise
except:
# Happens during interpreter shutdown
return
self.wait(0)
try:
mi, sequence = self.db.get_metadata_for_dump(book_id)
except:
prints('Failed to get backup metadata for id:', book_id, 'once')
traceback.print_exc()
self.wait(self.interval)
try:
mi, sequence = self.db.get_metadata_for_dump(book_id)
except:
prints('Failed to get backup metadata for id:', book_id,
'again, giving up')
traceback.print_exc()
return
if mi is None:
self.db.clear_dirtied(book_id, sequence)
return
# Give the GUI thread a chance to do something. Python threads don't
# have priorities, so this thread would naturally keep the processor
# until some scheduling event happens. The wait makes such an event
self.wait(self.scheduling_interval)
try:
raw = metadata_to_opf(mi)
except:
prints('Failed to convert to opf for id:', book_id)
traceback.print_exc()
self.db.clear_dirtied(book_id, sequence)
return
self.wait(self.scheduling_interval)
try:
self.db.write_backup(book_id, raw)
except:
prints('Failed to write backup metadata for id:', book_id, 'once')
traceback.print_exc()
self.wait(self.interval)
try:
self.db.write_backup(book_id, raw)
except:
prints('Failed to write backup metadata for id:', book_id,
'again, giving up')
traceback.print_exc()
return
self.db.clear_dirtied(book_id, sequence)
def break_cycles(self):
# Legacy compatibility
pass
def get_listed_chromecasts(
friendly_names=None,
uuids=None,
tries=None,
retry_wait=None,
timeout=None,
discovery_timeout=DISCOVER_TIMEOUT,
zeroconf_instance=None,
):
"""
Searches the network for chromecast devices matching a list of friendly
names or a list of UUIDs.
Returns a tuple of:
A list of Chromecast objects matching the criteria,
or an empty list if no matching chromecasts were found.
A service browser to keep the Chromecast mDNS data updated. When updates
are (no longer) needed, call browser.stop_discovery().
To only discover chromecast devices without connecting to them, use
discover_listed_chromecasts instead.
:param friendly_names: A list of wanted friendly names
:param uuids: A list of wanted uuids
:param tries: passed to get_chromecasts
:param retry_wait: passed to get_chromecasts
:param timeout: passed to get_chromecasts
:param discovery_timeout: A floating point number specifying the time to wait
devices matching the criteria have been found.
:param zeroconf_instance: An existing zeroconf instance.
"""
cc_list = {}
def add_callback(uuid, _service):
_LOGGER.debug("Found chromecast %s (%s)",
browser.devices[uuid].friendly_name, uuid)
def get_chromecast_from_uuid(uuid):
return get_chromecast_from_cast_info(
browser.devices[uuid],
zconf=zconf,
tries=tries,
retry_wait=retry_wait,
timeout=timeout,
)
friendly_name = browser.devices[uuid].friendly_name
try:
if uuids and uuid in uuids:
if uuid not in cc_list:
cc_list[uuid] = get_chromecast_from_uuid(uuid)
uuids.remove(uuid)
if friendly_names and friendly_name in friendly_names:
if uuid not in cc_list:
cc_list[uuid] = get_chromecast_from_uuid(uuid)
friendly_names.remove(friendly_name)
if not friendly_names and not uuids:
discover_complete.set()
except ChromecastConnectionError: # noqa: F405
pass
discover_complete = Event()
zconf = zeroconf_instance or zeroconf.Zeroconf()
browser = CastBrowser(SimpleCastListener(add_callback), zconf)
browser.start_discovery()
# Wait for the timeout or found all wanted devices
discover_complete.wait(discovery_timeout)
return (list(cc_list.values()), browser)
def sync(self, timeout=None):
evt = Event()
self.push(0, evt.set)
evt.wait(timeout)
class DHT:
def __init__(self,
node_id=None,
ip=None,
port=0,
password=None,
network_id="default",
debug=1,
networking=1):
self.node_id = node_id or self.rand_str(20)
if sys.version_info >= (3, 0, 0):
if type(self.node_id) == str:
self.node_id = self.node_id.encode("ascii")
else:
if type(self.node_id) == unicode:
self.node_id = str(self.node_id)
self.node_id = binascii.hexlify(self.node_id).decode('utf-8')
self.password = password or self.rand_str(30)
self.ip = ip
self.port = port
self.network_id = network_id
self.check_interval = 3 # For slow connections, unfortunately.
self.last_check = 0
self.debug = debug
self.networking = networking
self.relay_links = {}
self.protocol = DHTProtocol()
self.is_registered = Event()
self.is_mutex_ready = Event()
self.is_neighbours_ready = Event()
self.handles = []
self.threads = []
self.running = 1
self.has_mutex = 0
self.neighbours = []
# Register a new "account."
if self.networking:
self.register(self.node_id, self.password)
self.is_registered.wait(5)
self.mutex_loop()
self.is_mutex_ready.wait(5)
self.alive_loop()
self.find_neighbours_loop()
self.is_neighbours_ready.wait(5)
assert (self.is_mutex_ready.is_set())
assert (self.is_registered.is_set())
self.message_handlers = set()
def stop(self):
self.running = 0
for handle in self.handles:
handle.close()
# handle.raw._fp.close()
def hook_queue(self, q):
self.protocol.messages_received = q
self.check_for_new_messages()
def retry_in_thread(self, f, args={"args": None}, check_interval=2):
def thread_loop(this_obj):
while 1:
try:
while not f(**args) and this_obj.running:
time.sleep(check_interval)
if not this_obj.running:
return
return
except Exception as e:
print("unknown exception")
print(e)
time.sleep(1)
t = Thread(target=thread_loop, args=(self, ))
t.setDaemon(True)
self.threads.append(t)
t.start()
return t
def check_for_new_messages(self):
def do(args):
for msg in self.list(self.node_id, self.password):
self.protocol.messages_received.put(msg)
return 0
if LONG_POLLING:
self.retry_in_thread(do, check_interval=0.1)
else:
self.retry_in_thread(do, check_interval=2)
def mutex_loop(self):
def do(args):
# Requests a mutex from the server.
call = dht_msg_endpoint + "?call=get_mutex&"
call += urlencode({"node_id": self.node_id}) + "&"
call += urlencode({"password": self.password})
# Make API call.
ret = requests.get(call, timeout=5).text
if "1" in ret or "2" in ret:
self.has_mutex = int(ret)
self.is_mutex_ready.set()
return 0
self.retry_in_thread(do, check_interval=MUTEX_TIMEOUT)
def alive_loop(self):
def do(args):
# Requests a mutex from the server.
call = dht_msg_endpoint + "?call=last_alive&"
call += urlencode({"node_id": self.node_id}) + "&"
call += urlencode({"password": self.password})
# Make API call.
ret = requests.get(call, timeout=5)
return 0
self.retry_in_thread(do, check_interval=ALIVE_TIMEOUT)
def can_test_knode(self, id):
for neighbour in self.neighbours:
if neighbour.id == id:
if neighbour.can_test:
return 1
return 0
def has_testable_neighbours(self):
for neighbour in self.neighbours:
if neighbour.can_test:
return 1
return 0
def find_neighbours_loop(self):
def do(args):
# Requests a mutex from the server.
call = dht_msg_endpoint + "?call=find_neighbours&"
call += urlencode({"node_id": self.node_id}) + "&"
call += urlencode({"password": self.password}) + "&"
call += urlencode({"network_id": self.network_id})
# Make API call.
ret = requests.get(call, timeout=5).text
ret = json.loads(ret)
if type(ret) == dict:
ret = [ret]
# Convert to kademlia neighbours.
neighbours = []
for neighbour in ret:
if not is_ip_valid(neighbour["ip"]):
continue
neighbour["port"] = int(neighbour["port"])
if not is_valid_port(neighbour["port"]):
continue
knode = KadNode(id=binascii.unhexlify(
neighbour["id"].encode("ascii")),
ip=neighbour["ip"],
port=neighbour["port"],
can_test=int(neighbour["can_test"]))
neighbours.append(knode)
self.neighbours = neighbours
self.is_neighbours_ready.set()
return 0
self.retry_in_thread(do, check_interval=ALIVE_TIMEOUT)
def get_neighbours(self):
return self.neighbours
def add_relay_link(self, dht):
node_id = binascii.hexlify(dht.get_id())
self.relay_links[node_id.decode("utf-8")] = dht
def debug_print(self, msg):
if self.debug:
print(str(msg))
def add_message_handler(self, handler):
self.message_handlers.add(handler)
def remove_transfer_request_handler(self, handler):
pass
def rand_str(self, length):
return ''.join(
random.choice(string.digits + string.ascii_lowercase +
string.ascii_uppercase) for i in range(length))
def register(self, node_id, password):
def do(node_id, password):
try:
# Registers a new node to receive messages.
call = dht_msg_endpoint + "?call=register&"
call += urlencode({"node_id": node_id}) + "&"
call += urlencode({"password": password}) + "&"
call += urlencode({"port": self.port}) + "&"
call += urlencode({"network_id": self.network_id})
if self.ip is not None:
call += "&" + urlencode({"ip": self.ip})
# Make API call.
ret = requests.get(call, timeout=5)
self.handles.append(ret)
if "success" not in ret.text:
return 0
self.is_registered.set()
return 1
except Exception as e:
print(e)
self.debug_print("Register timed out in DHT msg")
self.debug_print("DHT REGISTER FAILED")
return 0
mappings = {"node_id": node_id, "password": password}
self.retry_in_thread(do, mappings)
def build_dht_response(self, msg):
msg = binascii.unhexlify(msg)
msg = umsgpack.unpackb(msg)
try:
str_types = [type(u""), type(b"")]
if type(msg) in str_types:
msg = literal_eval(msg)
except:
msg = str(msg)
return msg
def serialize_message(self, msg):
msg = umsgpack.packb(msg)
msg = binascii.hexlify(msg)
return msg
def async_dht_put(self, key, value):
d = defer.Deferred()
def do(args):
t = self.put(key, value, list_pop=0)
while t.isAlive():
time.sleep(1)
d.callback("success")
return 1
self.retry_in_thread(do)
return d
def async_dht_get(self, key):
d = defer.Deferred()
def do(args):
ret = self.list(node_id=key, list_pop=0, timeout=5)
if len(ret):
d.callback(ret[0])
else:
d.callback(None)
return 1
self.retry_in_thread(do)
return d
def put(self, node_id, msg, list_pop=1):
def do(node_id, msg):
if node_id in self.relay_links:
relay_link = self.relay_links[node_id]
msg = self.build_dht_response(self.serialize_message(msg))
relay_link.protocol.messages_received.put_nowait(msg)
return 1
try:
# Send a message directly to a node in the "DHT"
call = dht_msg_endpoint + "?call=put&"
call += urlencode({"dest_node_id": node_id}) + "&"
msg = self.serialize_message(msg)
call += urlencode({"msg": msg}) + "&"
call += urlencode({"node_id": self.node_id}) + "&"
call += urlencode({"password": self.password}) + "&"
call += urlencode({"list_pop": list_pop})
# Make API call.
ret = requests.get(call, timeout=5)
self.handles.append(ret)
if "success" not in ret.text:
return 0
return 1
except Exception as e:
# Reschedule call.
self.debug_print("DHT PUT TIMED OUT")
self.debug_print(e)
self.debug_print("Rescheduling DHT PUT")
self.debug_print("PUT FAILED")
return 0
mappings = {"node_id": node_id, "msg": msg}
return self.retry_in_thread(do, mappings)
def list(self, node_id=None, password=None, list_pop=1, timeout=None):
if not self.networking:
return []
node_id = node_id or self.node_id
password = password or self.password
try:
# Get messages send to us in the "DHT"
call = dht_msg_endpoint + "?call=list&"
call += urlencode({"node_id": node_id}) + "&"
call += urlencode({"password": password}) + "&"
call += urlencode({"list_pop": list_pop})
# Make API call.
if timeout is None:
if LONG_POLLING:
timeout = None
else:
timeout = 4
ret = requests.get(call, timeout=timeout)
self.handles.append(ret)
content_gen = ret.iter_content()
messages = ret.text
messages = json.loads(messages)
# List.
if type(messages) == dict:
messages = [messages]
# Return a list of responses.
ret = []
if type(messages) == list:
for msg in messages:
dht_response = self.build_dht_response(msg)
ret.append(dht_response)
return ret
except Exception as e:
print("EXCEPTION IN DHT MSG LIST")
self.debug_print("Exception in dht msg list")
print(e)
return []
def direct_message(self, node_id, msg):
return self.send_direct_message(node_id, msg)
def relay_message(self, node_id, msg):
return self.send_direct_message(node_id, msg)
def repeat_relay_message(self, node_id, msg):
return self.send_direct_message(node_id, msg)
def async_direct_message(self, node_id, msg):
return self.send_direct_message(node_id, msg)
def send_direct_message(self, node_id, msg):
if sys.version_info >= (3, 0, 0):
if type(node_id) == bytes:
node_id = binascii.hexlify(node_id).decode("utf-8")
else:
if type(node_id) == str:
node_id = binascii.hexlify(node_id).decode("utf-8")
if type(node_id) != str:
node_id = node_id.decode("utf-8")
self.put(node_id, msg)
def get_id(self):
node_id = self.node_id
if sys.version_info >= (3, 0, 0):
if type(node_id) == str:
node_id = node_id.encode("ascii")
else:
if type(node_id) == unicode:
node_id = str(node_id)
return binascii.unhexlify(node_id)
def has_messages(self):
return not self.protocol.messages_received.empty()
def get_messages(self):
result = []
if self.has_messages():
while not self.protocol.messages_received.empty():
result.append(self.protocol.messages_received.get())
# Run handlers on messages.
old_handlers = set()
for received in result:
for handler in self.message_handlers:
expiry = handler(self, received)
if expiry == -1:
old_handlers.add(handler)
# Expire old handlers.
for handler in old_handlers:
self.message_handlers.remove(handler)
return result
return result
class _agentApp(Thread):
def __init__(self, name, broker_url, heartbeat):
Thread.__init__(self)
self.notifier = _testNotifier()
self.broker_url = broker_url
self.agent = Agent(name,
_notifier=self.notifier,
heartbeat_interval=heartbeat)
# Management Database
# - two different schema packages,
# - two classes within one schema package
# - multiple objects per schema package+class
# - two "undescribed" objects
# "package1/class1"
_schema = SchemaObjectClass(_classId=SchemaClassId(
"package1", "class1"),
_desc="A test data schema - one",
_object_id_names=["key"])
_schema.add_property("key", SchemaProperty(qmfTypes.TYPE_LSTR))
_schema.add_property("count1", SchemaProperty(qmfTypes.TYPE_UINT32))
_schema.add_property("count2", SchemaProperty(qmfTypes.TYPE_UINT32))
self.agent.register_object_class(_schema)
_obj = QmfAgentData(self.agent,
_values={"key": "p1c1_key1"},
_schema=_schema)
_obj.set_value("count1", 0)
_obj.set_value("count2", 0)
self.agent.add_object(_obj)
_obj = QmfAgentData(self.agent,
_values={"key": "p1c1_key2"},
_schema=_schema)
_obj.set_value("count1", 9)
_obj.set_value("count2", 10)
self.agent.add_object(_obj)
# "package1/class2"
_schema = SchemaObjectClass(_classId=SchemaClassId(
"package1", "class2"),
_desc="A test data schema - two",
_object_id_names=["name"])
# add properties
_schema.add_property("name", SchemaProperty(qmfTypes.TYPE_LSTR))
_schema.add_property("string1", SchemaProperty(qmfTypes.TYPE_LSTR))
self.agent.register_object_class(_schema)
_obj = QmfAgentData(self.agent,
_values={"name": "p1c2_name1"},
_schema=_schema)
_obj.set_value("string1", "a data string")
self.agent.add_object(_obj)
# "package2/class1"
_schema = SchemaObjectClass(
_classId=SchemaClassId("package2", "class1"),
_desc="A test data schema - second package",
_object_id_names=["key"])
_schema.add_property("key", SchemaProperty(qmfTypes.TYPE_LSTR))
_schema.add_property("counter", SchemaProperty(qmfTypes.TYPE_UINT32))
self.agent.register_object_class(_schema)
_obj = QmfAgentData(self.agent,
_values={"key": "p2c1_key1"},
_schema=_schema)
_obj.set_value("counter", 0)
self.agent.add_object(_obj)
_obj = QmfAgentData(self.agent,
_values={"key": "p2c1_key2"},
_schema=_schema)
_obj.set_value("counter", 2112)
self.agent.add_object(_obj)
# add two "unstructured" objects to the Agent
_obj = QmfAgentData(self.agent, _object_id="undesc-1")
_obj.set_value("field1", "a value")
_obj.set_value("field2", 2)
_obj.set_value("field3", {"a": 1, "map": 2, "value": 3})
_obj.set_value("field4", ["a", "list", "value"])
self.agent.add_object(_obj)
_obj = QmfAgentData(self.agent, _object_id="undesc-2")
_obj.set_value("key-1", "a value")
_obj.set_value("key-2", 2)
self.agent.add_object(_obj)
self.running = False
self.ready = Event()
def start_app(self):
self.running = True
self.start()
self.ready.wait(10)
if not self.ready.is_set():
raise Exception("Agent failed to connect to broker.")
def stop_app(self):
self.running = False
self.notifier.indication() # hmmm... collide with daemon???
self.join(10)
if self.isAlive():
raise Exception("AGENT DID NOT TERMINATE AS EXPECTED!!!")
def run(self):
# broker_url = "user/passwd@hostname:port"
self.conn = qpid.messaging.Connection(self.broker_url)
self.conn.open()
self.agent.set_connection(self.conn)
self.ready.set()
while self.running:
self.notifier.wait_for_work(None)
wi = self.agent.get_next_workitem(timeout=0)
while wi is not None:
logging.error("UNEXPECTED AGENT WORKITEM RECEIVED=%s" %
wi.get_type())
self.agent.release_workitem(wi)
wi = self.agent.get_next_workitem(timeout=0)
if self.conn:
self.agent.remove_connection(10)
self.agent.destroy(10)
from threading import Thread,Event
from time import sleep
s = None #作为通信变量
e = Event()
def bar():
print("Bar 拜山头")
sleep(1)
global s
s = "天王盖地虎"
e.set()
b = Thread(target = bar)
b.start()
print("说对口令就是自己人")
e.wait() #阻塞等待，分支线程设置ｅ.set
if s == '天王盖地虎':
print("确认过眼神，你是对的人")
else:
print("打死他")
e.clear()
b.join()
class TimerQueue(Thread):
def __init__(self, daemon=False, time=monotonic):
super(TimerQueue, self).__init__()
self._time = time
self.daemon = daemon
self.done = False
self._evt = Event()
self._lock = Lock()
self._Q = []
def start(self):
super(TimerQueue, self).start()
return self
def join(self):
self.done = True
self._evt.set()
super(TimerQueue, self).join()
def sync(self, timeout=None):
evt = Event()
self.push(0, evt.set)
evt.wait(timeout)
def push(self, cb, delay=0):
_log.debug("Schedule %s %s", delay, cb)
deadline = self._time() + delay
ent = (deadline, cb)
with self._lock:
heapq.heappush(self._Q, ent)
wake = self._Q[0] is ent
if wake:
self._evt.set()
def run(self):
delay = None
while not self.done:
_log.debug("timer queue wait %s", delay)
if self._evt.wait(delay):
self._evt.clear()
delay = None
_log.debug("timer queue wake")
now = self._time()
actions = []
with self._lock:
while len(self._Q) and self._Q[0][0] <= now:
actions.append(heapq.heappop(self._Q)[1])
if len(self._Q):
delay = self._Q[0][
0] - now # TODO? biased by action execution time
for A in actions:
try:
A()
except:
_log.exception("Error in timer callback %s", A)
class Executor(AtomicSolver, CoupledSolver, Thread):
"""Associates a hierarchical DEVS model with the simulation engine.
"""
###
def __init__(self, model, sync=None):
"""Constructor.
{\tt model} is an instance of a valid hierarchical DEVS model. The
constructor stores a local reference to this model and augments it with
{\sl time variables\/} required by the simulator.
"""
Thread.__init__(self)
self.model = model
self.augment(self.model)
if sync == None:
from threading import Event
self.sync = Event()
else:
self.sync = sync
self.done = 0
###
def augment(self, d):
"""Recusively augment model {\tt d} with {\sl time variables\/}.
"""
# {\tt timeLast} holds the simulation time when the last event occured
# within the given DEVS, and (\tt myTimeAdvance} holds the amount of time until
# the next event.
d.timeLast = d.myTimeAdvance = 0.
if d.type() == 'COUPLED':
# {\tt eventList} is the list of pairs $(tn_d,\,d)$, where $d$ is a
# reference to a sub-model of the coupled-DEVS and $tn_d$ is $d$'s
# time of next event.
#d.eventList = []
d.immChildren = []
for subd in d.componentSet:
self.augment(subd)
###
def send(self, d, msg):
"""Dispatch messages to the right method.
"""
if d.type() == 'COUPLED':
return CoupledSolver.receive(self, d, msg)
elif d.type() == 'ATOMIC':
return AtomicSolver.receive(self, d, msg)
###
def shutdown(self):
self.done = 1
self.sync.set()
###
def run(self):
"""Simulate the model (Root-Coordinator).
In this implementation, the simulation stops only when the simulation
time (stored in {\tt clock}) reaches {\tt T} (starts at 0). Other means
to stop the simulation ({\it e.g.}, when an atomic-DEVS enters a given
state, or when a global variable reaches a predefined value) will be
implemented.
"""
# Initialize the model --- set the simulation clock to 0.
self.send(self.model, (0, [], 0))
self.model.extEvents = {}
from time import time
# initialRT = time()
clockRT = 0.0
# Main loop repeatedly sends $(*,\,t)$ messages to the model's root DEVS.
while 1:
# if the smallest time advance is infinity, wait forever
# if the smallest time advance is x, then wait(x)
# don't wait at all if the smallest time advance is 0
ta = self.model.myTimeAdvance
if ta == INFINITY:
initial = time()
self.sync.wait()
elapsed = time() - initial
elif ta - clockRT > 0:
initial = time()
self.sync.wait(ta - clockRT)
elapsed = time() - initial
else:
elapsed = 0.0
if ta == INFINITY or ta - clockRT > 0.0:
if self.sync.isSet(): clockRT += elapsed # INTERRUPT
else: clockRT += (ta - clockRT) # TIMEOUT
self.sync.clear()
if self.done:
return
# calculate how much time has passed so far
if __VERBOSE__:
print "\n", "* " * 10, "CLOCK (RT): %f" % clockRT
# pass incoming external events to their respective ports from the GUI
# if there are no external events then we must do an internal transition
immChildren = self.model.immChildren
external = 0
for inport, inport_name in self.model.IPorts:
for sourceport in inport.inLine:
if len(sourceport.host.myOutput) != 0:
external = 1
for e in sourceport.host.myOutput.values():
self.send(self.model, [{
inport: e
}, immChildren, clockRT])
sourceport.host.myOutput.clear()
if not external:
self.send(self.model, (1, immChildren, clockRT))
def run_browser(
command,
minidump_dir,
timeout=None,
on_started=None,
debug=None,
debugger=None,
debugger_args=None,
**kwargs
):
"""
Run the browser using the given `command`.
After the browser prints __endTimestamp, we give it 5
seconds to quit and kill it if it's still alive at that point.
Note that this method ensure that the process is killed at
the end. If this is not possible, an exception will be raised.
:param command: the commad (as a string list) to run the browser
:param minidump_dir: a path where to extract minidumps in case the
browser hang. This have to be the same value
used in `mozcrash.check_for_crashes`.
:param timeout: if specified, timeout to wait for the browser before
we raise a :class:`TalosError`
:param on_started: a callback that can be used to do things just after
the browser has been started. The callback must takes
an argument, which is the psutil.Process instance
:param kwargs: additional keyword arguments for the :class:`ProcessHandler`
instance
Returns a ProcessContext instance, with available output and pid used.
"""
debugger_info = find_debugger_info(debug, debugger, debugger_args)
if debugger_info is not None:
return run_in_debug_mode(
command, debugger_info, on_started=on_started, env=kwargs.get("env")
)
is_launcher = sys.platform.startswith("win") and "-wait-for-browser" in command
context = ProcessContext(is_launcher)
first_time = int(time.time()) * 1000
wait_for_quit_timeout = 20
event = Event()
reader = Reader(event)
LOG.info("Using env: %s" % pprint.pformat(kwargs["env"]))
kwargs["storeOutput"] = False
kwargs["processOutputLine"] = reader
kwargs["onFinish"] = event.set
proc = ProcessHandler(command, **kwargs)
reader.proc = proc
proc.run()
LOG.process_start(proc.pid, " ".join(command))
try:
context.process = psutil.Process(proc.pid)
if on_started:
on_started(context.process)
# wait until we saw __endTimestamp in the proc output,
# or the browser just terminated - or we have a timeout
if not event.wait(timeout):
LOG.info("Timeout waiting for test completion; killing browser...")
# try to extract the minidump stack if the browser hangs
kill_and_get_minidump(context, minidump_dir)
raise TalosError("timeout")
if reader.got_end_timestamp:
for i in six.moves.range(1, wait_for_quit_timeout):
if proc.wait(1) is not None:
break
if proc.poll() is None:
LOG.info(
"Browser shutdown timed out after {0} seconds, killing"
" process.".format(wait_for_quit_timeout)
)
kill_and_get_minidump(context, minidump_dir)
raise TalosError(
"Browser shutdown timed out after {0} seconds, killed"
" process.".format(wait_for_quit_timeout)
)
elif reader.got_timeout:
raise TalosError("TIMEOUT: %s" % reader.timeout_message)
elif reader.got_error:
raise TalosError("unexpected error")
finally:
# this also handle KeyboardInterrupt
# ensure early the process is really terminated
return_code = None
try:
return_code = context.kill_process()
if return_code is None:
return_code = proc.wait(1)
except Exception:
# Maybe killed by kill_and_get_minidump(), maybe ended?
LOG.info("Unable to kill process")
LOG.info(traceback.format_exc())
reader.output.append(
"__startBeforeLaunchTimestamp%d__endBeforeLaunchTimestamp" % first_time
)
reader.output.append(
"__startAfterTerminationTimestamp%d__endAfterTerminationTimestamp"
% (int(time.time()) * 1000)
)
if return_code is not None:
LOG.process_exit(proc.pid, return_code)
else:
LOG.debug("Unable to detect exit code of the process %s." % proc.pid)
context.output = reader.output
return context
class Scheduler(Thread):
"""The top level class of the task manager system.
The Scheduler is a thread that handles organizing and running tasks.
The Scheduler class should be instantiated to start a task manager session.
Its start method should be called to start the task manager.
Its stop method should be called to end the task manager session.
"""
## Init ##
def __init__(self, daemon=True, exceptionHandler=None):
Thread.__init__(self)
self._notifyEvent = Event()
self._nextTime = None
self._scheduled = {}
self._running = {}
self._onDemand = {}
self._isRunning = False
self._exceptionHandler = exceptionHandler
if daemon:
self.setDaemon(True)
## Event Methods ##
def wait(self, seconds=None):
"""Our own version of wait().
When called, it waits for the specified number of seconds, or until
it is notified that it needs to wake up, through the notify event.
"""
try:
self._notifyEvent.wait(seconds)
except IOError:
pass
self._notifyEvent.clear()
## Attributes ##
def runningTasks(self):
"""Return all running tasks."""
return self._running
def running(self, name, default=None):
"""Return running task with given name.
Returns a task with the given name from the "running" list,
if it is present there.
"""
return self._running.get(name, default)
def hasRunning(self, name):
"""Check to see if a task with the given name is currently running."""
return name in self._running
def setRunning(self, handle):
"""Add a task to the running dictionary.
Used internally only.
"""
self._running[handle.name()] = handle
def delRunning(self, name):
"""Delete a task from the running list.
Used internally.
"""
try:
handle = self._running[name]
del self._running[name]
return handle
except Exception:
return None
def scheduledTasks(self):
"""Return all scheduled tasks."""
return self._scheduled
def scheduled(self, name, default=None):
"""Return a task from the scheduled list."""
return self._scheduled.get(name, default)
def hasScheduled(self, name):
"""Checks whether task with given name is in the scheduled list."""
return name in self._scheduled
def setScheduled(self, handle):
"""Add the given task to the scheduled list."""
self._scheduled[handle.name()] = handle
def delScheduled(self, name):
"""Delete a task with the given name from the scheduled list."""
return self._scheduled.pop(name, None)
def onDemandTasks(self):
"""Return all on demand tasks."""
return self._onDemand
def onDemand(self, name, default=None):
"""Return a task from the onDemand list."""
return self._onDemand.get(name, default)
def hasOnDemand(self, name):
"""Checks whether task with given name is in the on demand list."""
return name in self._onDemand
def setOnDemand(self, handle):
"""Add the given task to the on demand list."""
self._onDemand[handle.name()] = handle
def delOnDemand(self, name):
"""Delete a task with the given name from the on demand list."""
return self._onDemand.pop(name, None)
def nextTime(self):
"""Get next execution time."""
return self._nextTime
def setNextTime(self, time):
"""Set next execution time."""
self._nextTime = time
def isRunning(self):
"""Check whether thread is running."""
return self._isRunning
## Adding Tasks ##
def addTimedAction(self, time, task, name):
"""Add a task to be run once, at a specific time."""
handle = self.unregisterTask(name)
if handle:
handle.reset(time, 0, task, True)
else:
handle = TaskHandler(self, time, 0, task, name)
self.scheduleTask(handle)
def addActionOnDemand(self, task, name):
"""Add a task to be run only on demand.
Adds a task to the scheduler that will not be scheduled
until specifically requested.
"""
handle = self.unregisterTask(name)
if handle:
handle.reset(time(), 0, task, True)
else:
handle = TaskHandler(self, time(), 0, task, name)
handle.setOnDemand()
self.setOnDemand(handle)
def addDailyAction(self, hour, minute, task, name):
"""Add an action to be run every day at a specific time.
If a task with the given name is already registered with the
scheduler, that task will be removed from the scheduling queue
and registered anew as a periodic task.
Can we make this addCalendarAction? What if we want to run
something once a week? We probably don't need that for Webware,
but this is a more generally useful module. This could be a
difficult function, though. Particularly without mxDateTime.
"""
current = localtime()
currHour = current[3]
currMin = current[4]
if hour > currHour:
hourDifference = hour - currHour
if minute > currMin:
minuteDifference = minute - currMin
elif minute < currMin:
minuteDifference = 60 - currMin + minute
hourDifference -= 1
else:
minuteDifference = 0
elif hour < currHour:
hourDifference = 24 - currHour + hour
if minute > currMin:
minuteDifference = minute - currMin
elif minute < currMin:
minuteDifference = 60 - currMin + minute
hourDifference -= 1
else:
minuteDifference = 0
else:
if minute > currMin:
hourDifference = 0
minuteDifference = minute - currMin
elif minute < currMin:
minuteDifference = 60 - currMin + minute
hourDifference = 23
else:
hourDifference = 0
minuteDifference = 0
delay = (minuteDifference + (hourDifference * 60)) * 60
self.addPeriodicAction(time() + delay, 24 * 60 * 60, task, name)
def addPeriodicAction(self, start, period, task, name):
"""Add a task to be run periodically.
Adds an action to be run at a specific initial time,
and every period thereafter.
The scheduler will not reschedule a task until the last
scheduled instance of the task has completed.
If a task with the given name is already registered with
the scheduler, that task will be removed from the scheduling
queue and registered anew as a periodic task.
"""
handle = self.unregisterTask(name)
if handle:
handle.reset(start, period, task, True)
else:
handle = TaskHandler(self, start, period, task, name)
self.scheduleTask(handle)
## Task methods ##
def unregisterTask(self, name):
"""Unregisters the named task.
After that it can be rescheduled with different parameters,
or simply removed.
"""
handle = (self.delRunning(name) or self.delScheduled(name)
or self.delOnDemand(name))
if handle:
handle.unregister()
return handle
def runTaskNow(self, name):
"""Allow a registered task to be immediately executed.
Returns True if the task is either currently running or was started,
or False if the task could not be found in the list of currently
registered tasks.
"""
if self.hasRunning(name):
return True
handle = self.scheduled(name)
if not handle:
handle = self.onDemand(name)
if not handle:
return False
self.runTask(handle)
return True
def demandTask(self, name):
"""Demand execution of a task.
Allow the server to request that a task listed as being registered
on-demand be run as soon as possible.
If the task is currently running, it will be flagged to run again
as soon as the current run completes.
Returns False if the task name could not be found on the on-demand
or currently running lists.
"""
if self.hasRunning(name) or self.hasOnDemand(name):
handle = self.running(name)
if handle:
handle.runOnCompletion()
return True
handle = self.onDemand(name)
if not handle:
return False
self.runTask(handle)
return True
else:
return False
def stopTask(self, name):
"""Put an immediate halt to a running background task.
Returns True if the task was either not running, or was
running and was told to stop.
"""
handle = self.running(name)
if not handle:
return False
handle.stop()
return True
def stopAllTasks(self):
"""Terminate all running tasks."""
for i in self._running:
self.stopTask(i)
def disableTask(self, name):
"""Specify that a task be suspended.
Suspended tasks will not be scheduled until later enabled.
If the task is currently running, it will not be interfered
with, but the task will not be scheduled for execution in
future until re-enabled.
Returns True if the task was found and disabled.
"""
handle = self.running(name)
if not handle:
handle = self.scheduled(name)
if not handle:
return False
handle.disable()
return True
def enableTask(self, name):
"""Enable a task again.
This method is provided to specify that a task be re-enabled
after a suspension. A re-enabled task will be scheduled for
execution according to its original schedule, with any runtimes
that would have been issued during the time the task was suspended
simply skipped.
Returns True if the task was found and enabled.
"""
handle = self.running(name)
if not handle:
handle = self.scheduled(name)
if not handle:
return False
handle.enable()
return True
def runTask(self, handle):
"""Run a task.
Used by the Scheduler thread's main loop to put a task in
the scheduled hash onto the run hash.
"""
name = handle.name()
if self.delScheduled(name) or self.delOnDemand(name):
self.setRunning(handle)
handle.runTask()
def scheduleTask(self, handle):
"""Schedule a task.
This method takes a task that needs to be scheduled and adds it
to the scheduler. All scheduling additions or changes are handled
by this method. This is the only Scheduler method that can notify
the run() method that it may need to wake up early to handle a
newly registered task.
"""
self.setScheduled(handle)
if not self.nextTime() or handle.startTime() < self.nextTime():
self.setNextTime(handle.startTime())
self.notify()
## Misc Methods ##
def notifyCompletion(self, handle):
"""Notify completion of a task.
Used by instances of TaskHandler to let the Scheduler thread know
when their tasks have run to completion. This method is responsible
for rescheduling the task if it is a periodic task.
"""
name = handle.name()
if self.hasRunning(name):
self.delRunning(name)
if handle.startTime() and handle.startTime() > time():
self.scheduleTask(handle)
else:
if handle.reschedule():
self.scheduleTask(handle)
elif handle.isOnDemand():
self.setOnDemand(handle)
if handle.runAgain():
self.runTask(handle)
def notifyFailure(self, handle):
"""Notify failure of a task.
Used by instances of TaskHandler to let the Scheduler thread know
if an exception has occurred within the task thread.
"""
self.notifyCompletion(handle)
if self._exceptionHandler is not None:
self._exceptionHandler()
def notify(self):
"""Wake up scheduler by sending a notify even."""
self._notifyEvent.set()
def start(self):
"""Start the scheduler's activity."""
self._isRunning = True
Thread.start(self)
def stop(self):
"""Terminate the scheduler and its associated tasks."""
self._isRunning = False
self.notify()
self.stopAllTasks()
# jdh: wait until the scheduler thread exits; otherwise
# it's possible for the interpreter to exit before this thread
# has a chance to shut down completely, which causes a traceback
# cz: but use a timeout of 3 seconds, this should suffice
self.join(3)
## Main Method ##
def run(self):
"""The main method of the scheduler running as a background thread.
This method is responsible for carrying out the scheduling work of
this class on a background thread. The basic logic is to wait until
the next action is due to run, move the task from our scheduled
list to our running list, and run it. Other synchronized methods
such as runTask(), scheduleTask(), and notifyCompletion(), may
be called while this method is waiting for something to happen.
These methods modify the data structures that run() uses to
determine its scheduling needs.
"""
while self._isRunning:
if self.nextTime():
nextTime = self.nextTime()
currentTime = time()
if currentTime < nextTime:
sleepTime = nextTime - currentTime
self.wait(sleepTime)
if not self._isRunning:
return
currentTime = time()
if currentTime >= nextTime:
toRun = []
nextRun = None
for handle in self._scheduled.values():
startTime = handle.startTime()
if startTime <= currentTime:
toRun.append(handle)
else:
if not nextRun:
nextRun = startTime
elif startTime < nextRun:
nextRun = startTime
self.setNextTime(nextRun)
for handle in toRun:
self.runTask(handle)
else:
self.wait()
logger.info("customer:{} push {} results".format(
str(request.args[0]), result))
def exception_alarm(request, exc_info):
logger.error("customer:{} push_failed. {}".format(
request.args[0], exc_info))
# send "push_failed" event to monitor
logger.event("push_failed",
"customer:{} {}".format(request.args[0], exc_info),
errorcode='01140509')
pool = threadpool.ThreadPool(POOLSIZE)
try:
while True:
kev.wait()
customers = dao.get_all_customers()
logger.debug("customers is {}".format(str(customers)))
requests = threadpool.makeRequests(do_push, customers, log_result,
exception_alarm)
for req in requests:
pool.putRequest(req)
pool.wait()
time.sleep(LOOP_INTERVAL)
if dbpc_thr:
dbpc_thr.join()
master_thr.join()
except:
error_trace = traceback.format_exc()
logger.error("I catch unknown error, exit!", exc_info=True)
class JobQueue(object):
"""This class allows you to periodically perform tasks with the bot.
Attributes:
queue (PriorityQueue):
bot (Bot):
Args:
bot (Bot): The bot instance that should be passed to the jobs
Deprecated: 5.2
prevent_autostart (Optional[bool]): Thread does not start during initialisation.
Use `start` method instead.
"""
def __init__(self, bot, prevent_autostart=None):
if prevent_autostart is not None:
warnings.warn(
"prevent_autostart is being deprecated, use `start` method instead."
)
self.queue = PriorityQueue()
self.bot = bot
self.logger = logging.getLogger(self.__class__.__name__)
self.__start_lock = Lock()
self.__next_peek_lock = Lock(
) # to protect self._next_peek & self.__tick
self.__tick = Event()
self.__thread = None
""":type: Thread"""
self._next_peek = None
""":type: float"""
self._running = False
def put(self, job, next_t=None):
"""Queue a new job.
Args:
job (Job): The ``Job`` instance representing the new job
next_t (Optional[float]): Time in seconds in which the job should be executed first.
Defaults to ``job.interval``
"""
job.job_queue = self
if next_t is None:
next_t = job.interval
now = time.time()
next_t += now
self.logger.debug('Putting job %s with t=%f', job.name, next_t)
self.queue.put((next_t, job))
# Wake up the loop if this job should be executed next
self._set_next_peek(next_t)
def _set_next_peek(self, t):
"""
Set next peek if not defined or `t` is before next peek.
In case the next peek was set, also trigger the `self.__tick` event.
"""
with self.__next_peek_lock:
if not self._next_peek or self._next_peek > t:
self._next_peek = t
self.__tick.set()
def tick(self):
"""
Run all jobs that are due and re-enqueue them with their interval.
"""
now = time.time()
self.logger.debug('Ticking jobs with t=%f', now)
while True:
try:
t, job = self.queue.get(False)
except Empty:
break
self.logger.debug('Peeked at %s with t=%f', job.name, t)
if t > now:
# we can get here in two conditions:
# 1. At the second or later pass of the while loop, after we've already processed
# the job(s) we were supposed to at this time.
# 2. At the first iteration of the loop only if `self.put()` had triggered
# `self.__tick` because `self._next_peek` wasn't set
self.logger.debug("Next task isn't due yet. Finished!")
self.queue.put((t, job))
self._set_next_peek(t)
break
if job._remove.is_set():
self.logger.debug('Removing job %s', job.name)
continue
if job.enabled:
self.logger.debug('Running job %s', job.name)
try:
job.run(self.bot)
except:
self.logger.exception(
'An uncaught error was raised while executing job %s',
job.name)
else:
self.logger.debug('Skipping disabled job %s', job.name)
if job.repeat:
self.put(job)
def start(self):
"""
Starts the job_queue thread.
"""
self.__start_lock.acquire()
if not self._running:
self._running = True
self.__start_lock.release()
self.__thread = Thread(target=self._main_loop, name="job_queue")
self.__thread.start()
self.logger.debug('%s thread started', self.__class__.__name__)
else:
self.__start_lock.release()
def _main_loop(self):
"""
Thread target of thread ``job_queue``. Runs in background and performs ticks on the job
queue.
"""
while self._running:
# self._next_peek may be (re)scheduled during self.tick() or self.put()
with self.__next_peek_lock:
tmout = self._next_peek and self._next_peek - time.time()
self._next_peek = None
self.__tick.clear()
self.__tick.wait(tmout)
# If we were woken up by self.stop(), just bail out
if not self._running:
break
self.tick()
self.logger.debug('%s thread stopped', self.__class__.__name__)
def stop(self):
"""
Stops the thread
"""
with self.__start_lock:
self._running = False
self.__tick.set()
if self.__thread is not None:
self.__thread.join()
def jobs(self):
"""Returns a tuple of all jobs that are currently in the ``JobQueue``"""
return tuple(job[1] for job in self.queue.queue if job)
class WebsocketClient(object):
def __init__(self, host=None, port=None, route=None, ssl=None):
config = Configuration.get().get("websocket")
host = host or config.get("host")
port = port or config.get("port")
route = route or config.get("route")
ssl = ssl or config.get("ssl")
validate_param(host, "websocket.host")
validate_param(port, "websocket.port")
validate_param(route, "websocket.route")
self.url = WebsocketClient.build_url(host, port, route, ssl)
self.emitter = EventEmitter()
self.client = self.create_client()
self.pool = ThreadPool(10)
self.retry = 5
self.connected_event = Event()
self.started_running = False
@staticmethod
def build_url(host, port, route, ssl):
scheme = "wss" if ssl else "ws"
return scheme + "://" + host + ":" + str(port) + route
def create_client(self):
return WebSocketApp(self.url,
on_open=self.on_open,
on_close=self.on_close,
on_error=self.on_error,
on_message=self.on_message)
def on_open(self, ws):
LOG.info("Connected")
self.connected_event.set()
self.emitter.emit("open")
# Restore reconnect timer to 5 seconds on sucessful connect
self.retry = 5
def on_close(self, ws):
self.emitter.emit("close")
def on_error(self, ws, error):
""" On error start trying to reconnect to the websocket. """
if isinstance(error, WebSocketConnectionClosedException):
LOG.warning('Could not send message because connection has closed')
else:
LOG.exception('=== ' + repr(error) + ' ===')
try:
self.emitter.emit('error', error)
if self.client.keep_running:
self.client.close()
except Exception as e:
LOG.error('Exception closing websocket: ' + repr(e))
LOG.warning("WS Client will reconnect in %d seconds." % self.retry)
time.sleep(self.retry)
self.retry = min(self.retry * 2, 60)
try:
self.client = self.create_client()
self.run_forever()
except WebSocketException:
pass
def on_message(self, ws, message):
self.emitter.emit('message', message)
parsed_message = Message.deserialize(message)
self.pool.apply_async(self.emitter.emit,
(parsed_message.type, parsed_message))
def emit(self, message):
if not self.connected_event.wait(10):
if not self.started_running:
raise ValueError('You must execute run_forever() '
'before emitting messages')
self.connected_event.wait()
try:
if hasattr(message, 'serialize'):
self.client.send(message.serialize())
else:
self.client.send(json.dumps(message.__dict__))
except WebSocketConnectionClosedException:
LOG.warning('Could not send {} message because connection '
'has been closed'.format(message.type))
def wait_for_response(self, message, reply_type=None, timeout=None):
"""Send a message and wait for a response.
Args:
message (Message): message to send
reply_type (str): the message type of the expected reply.
Defaults to "<message.type>.response".
timeout: seconds to wait before timeout, defaults to 3
Returns:
The received message or None if the response timed out
"""
response = []
def handler(message):
"""Receive response data."""
response.append(message)
# Setup response handler
self.once(reply_type or message.type + '.response', handler)
# Send request
self.emit(message)
# Wait for response
start_time = time.monotonic()
while len(response) == 0:
time.sleep(0.2)
if time.monotonic() - start_time > (timeout or 3.0):
try:
self.remove(reply_type, handler)
except (ValueError, KeyError):
# ValueError occurs on pyee 1.0.1 removing handlers
# registered with once.
# KeyError may theoretically occur if the event occurs as
# the handler is removbed
pass
return None
return response[0]
def on(self, event_name, func):
self.emitter.on(event_name, func)
def once(self, event_name, func):
self.emitter.once(event_name, func)
def remove(self, event_name, func):
try:
self.emitter.remove_listener(event_name, func)
except ValueError as e:
LOG.warning('Failed to remove event {}: {}'.format(event_name, e))
def remove_all_listeners(self, event_name):
'''
Remove all listeners connected to event_name.
Args:
event_name: event from which to remove listeners
'''
if event_name is None:
raise ValueError
self.emitter.remove_all_listeners(event_name)
def run_forever(self):
self.started_running = True
self.client.run_forever()
def close(self):
self.client.close()
self.connected_event.clear()
class Py3statusWrapper:
"""
This is the py3status wrapper.
"""
def __init__(self):
"""
Useful variables we'll need.
"""
self.config = {}
self.i3bar_running = True
self.last_refresh_ts = time.time()
self.lock = Event()
self.modules = {}
self.notified_messages = set()
self.output_modules = {}
self.py3_modules = []
self.py3_modules_initialized = False
self.running = True
self.update_queue = deque()
self.update_request = Event()
# shared code
common = Common(self)
self.get_config_attribute = common.get_config_attribute
self.report_exception = common.report_exception
# these are used to schedule module updates
self.timeout_update_due = deque()
self.timeout_queue = {}
self.timeout_queue_lookup = {}
self.timeout_keys = []
def timeout_queue_add_module(self, module, cache_time=0):
"""
Add a module to the timeout_queue if it is scheduled in the future or
if it is due for an update imediately just trigger that.
the timeout_queue is a dict with the scheduled time as the key and the
value is a list of module instance names due to be updated at that
point. An ordered list of keys is kept to allow easy checking of when
updates are due. A list is also kept of which modules are in the
update_queue to save having to search for modules in it unless needed.
"""
# If already set to update do nothing
if module in self.timeout_update_due:
return
# remove if already in the queue
key = self.timeout_queue_lookup.get(module)
if key:
try:
queue_item = self.timeout_queue[key]
try:
queue_item.remove(module)
except KeyError:
pass
if not queue_item:
del self.timeout_queue[key]
self.timeout_keys.remove(key)
except KeyError:
pass
if cache_time == 0:
# if cache_time is 0 we can just trigger the module update
self.timeout_update_due.append(module)
self.update_request.set()
else:
# add the module to the timeout queue
if cache_time not in self.timeout_keys:
self.timeout_queue[cache_time] = set([module])
self.timeout_keys.append(cache_time)
# sort keys so earliest is first
self.timeout_keys.sort()
else:
self.timeout_queue[cache_time].add(module)
# note that the module is in the timeout_queue
self.timeout_queue_lookup[module] = cache_time
def timeout_queue_process(self):
"""
Check the timeout_queue and set any due modules to update.
"""
now = time.time()
due_timeouts = []
# find any due timeouts
for timeout in self.timeout_keys:
if timeout > now:
break
due_timeouts.append(timeout)
# process them
for timeout in due_timeouts:
modules = self.timeout_queue[timeout]
# remove from the queue
del self.timeout_queue[timeout]
self.timeout_keys.remove(timeout)
for module in modules:
# module no longer in queue
del self.timeout_queue_lookup[module]
# tell module to update
self.timeout_update_due.append(module)
# run any modules that are due
while self.timeout_update_due:
module = self.timeout_update_due.popleft()
if isinstance(module, Module):
r = Runner(module, self)
r.start()
else:
# i3status module
module.update()
# we return how long till we next need to process the timeout_queue
try:
return self.timeout_keys[0] - time.time()
except IndexError:
return None
def get_config(self):
"""
Create the py3status based on command line options we received.
"""
# get home path
home_path = os.path.expanduser('~')
# defaults
config = {
'cache_timeout': 60,
'interval': 1,
'minimum_interval': 0.1, # minimum module update interval
'dbus_notify': False,
}
# include path to search for user modules
config['include_paths'] = [
'{}/.i3/py3status/'.format(home_path),
'{}/i3status/py3status'.format(
os.environ.get('XDG_CONFIG_HOME',
'{}/.config'.format(home_path))),
'{}/i3/py3status'.format(
os.environ.get('XDG_CONFIG_HOME',
'{}/.config'.format(home_path))),
]
config['version'] = version
# i3status config file default detection
# respect i3status' file detection order wrt issue #43
i3status_config_file_candidates = [
'{}/.i3status.conf'.format(home_path), '{}/i3status/config'.format(
os.environ.get('XDG_CONFIG_HOME',
'{}/.config'.format(home_path))),
'/etc/i3status.conf', '{}/i3status/config'.format(
os.environ.get('XDG_CONFIG_DIRS', '/etc/xdg'))
]
for fn in i3status_config_file_candidates:
if os.path.isfile(fn):
i3status_config_file_default = fn
break
else:
# if none of the default files exists, we will default
# to ~/.i3/i3status.conf
i3status_config_file_default = '{}/.i3/i3status.conf'.format(
home_path)
# command line options
parser = argparse.ArgumentParser(
description='The agile, python-powered, i3status wrapper')
parser = argparse.ArgumentParser(add_help=True)
parser.add_argument('-b',
'--dbus-notify',
action="store_true",
default=False,
dest="dbus_notify",
help="""use notify-send to send user notifications
rather than i3-nagbar,
requires a notification daemon eg dunst""")
parser.add_argument('-c',
'--config',
action="store",
dest="i3status_conf",
type=str,
default=i3status_config_file_default,
help="path to i3status config file")
parser.add_argument('-d',
'--debug',
action="store_true",
help="be verbose in syslog")
parser.add_argument('-i',
'--include',
action="append",
dest="include_paths",
help="""include user-written modules from those
directories (default ~/.i3/py3status)""")
parser.add_argument('-l',
'--log-file',
action="store",
dest="log_file",
type=str,
default=None,
help="path to py3status log file")
parser.add_argument('-n',
'--interval',
action="store",
dest="interval",
type=float,
default=config['interval'],
help="update interval in seconds (default 1 sec)")
parser.add_argument('-s',
'--standalone',
action="store_true",
help="standalone mode, do not use i3status")
parser.add_argument('-t',
'--timeout',
action="store",
dest="cache_timeout",
type=int,
default=config['cache_timeout'],
help="""default injection cache timeout in seconds
(default 60 sec)""")
parser.add_argument('-v',
'--version',
action="store_true",
help="""show py3status version and exit""")
parser.add_argument('cli_command', nargs='*', help=argparse.SUPPRESS)
options = parser.parse_args()
if options.cli_command:
config['cli_command'] = options.cli_command
# only asked for version
if options.version:
print('py3status version {} (python {})'.format(
config['version'], python_version()))
sys.exit(0)
# override configuration and helper variables
config['cache_timeout'] = options.cache_timeout
config['debug'] = options.debug
config['dbus_notify'] = options.dbus_notify
if options.include_paths:
config['include_paths'] = options.include_paths
config['interval'] = int(options.interval)
config['log_file'] = options.log_file
config['standalone'] = options.standalone
config['i3status_config_path'] = options.i3status_conf
# all done
return config
def get_user_modules(self):
"""
Search configured include directories for user provided modules.
user_modules: {
'weather_yahoo': ('~/i3/py3status/', 'weather_yahoo.py')
}
"""
user_modules = {}
for include_path in self.config['include_paths']:
include_path = os.path.abspath(include_path) + '/'
if not os.path.isdir(include_path):
continue
for f_name in sorted(os.listdir(include_path)):
if not f_name.endswith('.py'):
continue
module_name = f_name[:-3]
# do not overwrite modules if already found
if module_name in user_modules:
pass
user_modules[module_name] = (include_path, f_name)
return user_modules
def get_user_configured_modules(self):
"""
Get a dict of all available and configured py3status modules
in the user's i3status.conf.
"""
user_modules = {}
if not self.py3_modules:
return user_modules
for module_name, module_info in self.get_user_modules().items():
for module in self.py3_modules:
if module_name == module.split(' ')[0]:
include_path, f_name = module_info
user_modules[module_name] = (include_path, f_name)
return user_modules
def load_modules(self, modules_list, user_modules):
"""
Load the given modules from the list (contains instance name) with
respect to the user provided modules dict.
modules_list: ['weather_yahoo paris', 'net_rate']
user_modules: {
'weather_yahoo': ('/etc/py3status.d/', 'weather_yahoo.py')
}
"""
for module in modules_list:
# ignore already provided modules (prevents double inclusion)
if module in self.modules:
continue
try:
my_m = Module(module, user_modules, self)
# only handle modules with available methods
if my_m.methods:
self.modules[module] = my_m
elif self.config['debug']:
self.log('ignoring module "{}" (no methods found)'.format(
module))
except Exception:
err = sys.exc_info()[1]
msg = 'Loading module "{}" failed ({}).'.format(module, err)
self.report_exception(msg, level='warning')
def setup(self):
"""
Setup py3status and spawn i3status/events/modules threads.
"""
# SIGTSTP will be received from i3bar indicating that all output should
# stop and we should consider py3status suspended. It is however
# important that any processes using i3 ipc should continue to receive
# those events otherwise it can lead to a stall in i3.
signal(SIGTSTP, self.i3bar_stop)
# SIGCONT indicates output should be resumed.
signal(SIGCONT, self.i3bar_start)
# update configuration
self.config.update(self.get_config())
if self.config.get('cli_command'):
self.handle_cli_command(self.config)
sys.exit()
# logging functionality now available
# log py3status and python versions
self.log('=' * 8)
self.log('Starting py3status version {} python {}'.format(
self.config['version'], python_version()))
try:
# if running from git then log the branch and last commit
# we do this by looking in the .git directory
git_path = os.path.join(os.path.dirname(__file__), '..', '.git')
# branch
with open(os.path.join(git_path, 'HEAD'), 'r') as f:
out = f.readline()
branch = '/'.join(out.strip().split('/')[2:])
self.log('git branch: {}'.format(branch))
# last commit
log_path = os.path.join(git_path, 'logs', 'refs', 'heads', branch)
with open(log_path, 'r') as f:
out = f.readlines()[-1]
sha = out.split(' ')[1][:7]
msg = ':'.join(out.strip().split('\t')[-1].split(':')[1:])
self.log('git commit: {}{}'.format(sha, msg))
except:
pass
if self.config['debug']:
self.log('py3status started with config {}'.format(self.config))
# read i3status.conf
config_path = self.config['i3status_config_path']
self.config['py3_config'] = process_config(config_path, self)
# setup i3status thread
self.i3status_thread = I3status(self)
# If standalone or no i3status modules then use the mock i3status
# else start i3status thread.
i3s_modules = self.config['py3_config']['i3s_modules']
if self.config['standalone'] or not i3s_modules:
self.i3status_thread.mock()
i3s_mode = 'mocked'
else:
i3s_mode = 'started'
self.i3status_thread.start()
while not self.i3status_thread.ready:
if not self.i3status_thread.is_alive():
# i3status is having a bad day, so tell the user what went
# wrong and do the best we can with just py3status modules.
err = self.i3status_thread.error
self.notify_user(err)
self.i3status_thread.mock()
i3s_mode = 'mocked'
break
time.sleep(0.1)
if self.config['debug']:
self.log('i3status thread {} with config {}'.format(
i3s_mode, self.config['py3_config']))
# setup input events thread
self.events_thread = Events(self)
self.events_thread.daemon = True
self.events_thread.start()
if self.config['debug']:
self.log('events thread started')
# initialise the command server
self.commands_thread = CommandServer(self)
self.commands_thread.daemon = True
self.commands_thread.start()
if self.config['debug']:
self.log('commands thread started')
# suppress modules' ouput wrt issue #20
if not self.config['debug']:
sys.stdout = open('/dev/null', 'w')
sys.stderr = open('/dev/null', 'w')
# get the list of py3status configured modules
self.py3_modules = self.config['py3_config']['py3_modules']
# get a dict of all user provided modules
user_modules = self.get_user_configured_modules()
if self.config['debug']:
self.log('user_modules={}'.format(user_modules))
if self.py3_modules:
# load and spawn i3status.conf configured modules threads
self.load_modules(self.py3_modules, user_modules)
def notify_user(self, msg, level='error', rate_limit=None, module_name=''):
"""
Display notification to user via i3-nagbar or send-notify
We also make sure to log anything to keep trace of it.
NOTE: Message should end with a '.' for consistency.
"""
dbus = self.config.get('dbus_notify')
if dbus:
# force msg to be a string
msg = u'{}'.format(msg)
else:
msg = u'py3status: {}'.format(msg)
if level != 'info' and module_name == '':
fix_msg = u'{} Please try to fix this and reload i3wm (Mod+Shift+R)'
msg = fix_msg.format(msg)
# Rate limiting. If rate limiting then we need to calculate the time
# period for which the message should not be repeated. We just use
# A simple chunked time model where a message cannot be repeated in a
# given time period. Messages can be repeated more frequently but must
# be in different time periods.
limit_key = ''
if rate_limit:
try:
limit_key = time.time() // rate_limit
except TypeError:
pass
# We use a hash to see if the message is being repeated. This is crude
# and imperfect but should work for our needs.
msg_hash = hash(u'{}#{}#{}'.format(module_name, limit_key, msg))
if msg_hash in self.notified_messages:
return
else:
self.log(msg, level)
self.notified_messages.add(msg_hash)
try:
if dbus:
# fix any html entities
msg = msg.replace('&', '&')
msg = msg.replace('<', '<')
msg = msg.replace('>', '>')
cmd = [
'notify-send', '-u',
DBUS_LEVELS.get(level, 'normal'), '-t', '10000',
'py3status', msg
]
else:
py3_config = self.config['py3_config']
nagbar_font = py3_config.get('py3status').get('nagbar_font')
if nagbar_font:
cmd = [
'i3-nagbar', '-f', nagbar_font, '-m', msg, '-t', level
]
else:
cmd = ['i3-nagbar', '-m', msg, '-t', level]
Popen(cmd,
stdout=open('/dev/null', 'w'),
stderr=open('/dev/null', 'w'))
except:
pass
def stop(self):
"""
Set the Event lock, this will break all threads' loops.
"""
self.running = False
# stop the command server
try:
self.commands_thread.kill()
except:
pass
try:
self.lock.set()
if self.config['debug']:
self.log('lock set, exiting')
# run kill() method on all py3status modules
for module in self.modules.values():
module.kill()
except:
pass
def refresh_modules(self, module_string=None, exact=True):
"""
Update modules.
if module_string is None all modules are refreshed
if module_string then modules with the exact name or those starting
with the given string depending on exact parameter will be refreshed.
If a module is an i3status one then we refresh i3status.
To prevent abuse, we rate limit this function to 100ms for full
refreshes.
"""
if not module_string:
if time.time() > (self.last_refresh_ts + 0.1):
self.last_refresh_ts = time.time()
else:
# rate limiting
return
update_i3status = False
for name, module in self.output_modules.items():
if (module_string is None or (exact and name == module_string)
or (not exact and name.startswith(module_string))):
if module['type'] == 'py3status':
if self.config['debug']:
self.log('refresh py3status module {}'.format(name))
module['module'].force_update()
else:
if self.config['debug']:
self.log('refresh i3status module {}'.format(name))
update_i3status = True
if update_i3status:
self.i3status_thread.refresh_i3status()
def sig_handler(self, signum, frame):
"""
SIGUSR1 was received, the user asks for an immediate refresh of the bar
"""
self.log('received USR1')
self.refresh_modules()
def terminate(self, signum, frame):
"""
Received request to terminate (SIGTERM), exit nicely.
"""
raise KeyboardInterrupt()
def purge_module(self, module_name):
"""
A module has been removed e.g. a module that had an error.
We need to find any containers and remove the module from them.
"""
containers = self.config['py3_config']['.module_groups']
containers_to_update = set()
if module_name in containers:
containers_to_update.update(set(containers[module_name]))
for container in containers_to_update:
try:
self.modules[container].module_class.items.remove(module_name)
except ValueError:
pass
def notify_update(self, update, urgent=False):
"""
Name or list of names of modules that have updated.
"""
if not isinstance(update, list):
update = [update]
self.update_queue.extend(update)
# if all our py3status modules are not ready to receive updates then we
# don't want to get them to update.
if not self.py3_modules_initialized:
return
# find containers that use the modules that updated
containers = self.config['py3_config']['.module_groups']
containers_to_update = set()
for item in update:
if item in containers:
containers_to_update.update(set(containers[item]))
# force containers to update
for container in containers_to_update:
container_module = self.output_modules.get(container)
if container_module:
# If the container registered a urgent_function then call it
# if this update is urgent.
if urgent and container_module.get('urgent_function'):
container_module['urgent_function'](update)
# If a container has registered a content_function we use that
# to see if the container needs to be updated.
# We only need to update containers if their active content has
# changed.
if container_module.get('content_function'):
if set(update) & container_module['content_function']():
container_module['module'].force_update()
else:
# we don't know so just update.
container_module['module'].force_update()
# we need to update the output
if self.update_queue:
self.update_request.set()
def log(self, msg, level='info'):
"""
log this information to syslog or user provided logfile.
"""
if not self.config.get('log_file'):
# If level was given as a str then convert to actual level
level = LOG_LEVELS.get(level, level)
syslog(level, u'{}'.format(msg))
else:
# Binary mode so fs encoding setting is not an issue
with open(self.config['log_file'], 'ab') as f:
log_time = time.strftime("%Y-%m-%d %H:%M:%S")
# nice formating of data structures using pretty print
if isinstance(msg, (dict, list, set, tuple)):
msg = pformat(msg)
# if multiline then start the data output on a fresh line
# to aid readability.
if '\n' in msg:
msg = u'\n' + msg
out = u'{} {} {}\n'.format(log_time, level.upper(), msg)
try:
# Encode unicode strings to bytes
f.write(out.encode('utf-8'))
except (AttributeError, UnicodeDecodeError):
# Write any byte strings straight to log
f.write(out)
def create_output_modules(self):
"""
Setup our output modules to allow easy updating of py3modules and
i3status modules allows the same module to be used multiple times.
"""
py3_config = self.config['py3_config']
i3modules = self.i3status_thread.i3modules
output_modules = self.output_modules
# position in the bar of the modules
positions = {}
for index, name in enumerate(py3_config['order']):
if name not in positions:
positions[name] = []
positions[name].append(index)
# py3status modules
for name in self.modules:
if name not in output_modules:
output_modules[name] = {}
output_modules[name]['position'] = positions.get(name, [])
output_modules[name]['module'] = self.modules[name]
output_modules[name]['type'] = 'py3status'
# i3status modules
for name in i3modules:
if name not in output_modules:
output_modules[name] = {}
output_modules[name]['position'] = positions.get(name, [])
output_modules[name]['module'] = i3modules[name]
output_modules[name]['type'] = 'i3status'
self.output_modules = output_modules
def create_mappings(self, config):
"""
Create any mappings needed for global substitutions eg. colors
"""
mappings = {}
for name, cfg in config.items():
# Ignore special config sections.
if name in CONFIG_SPECIAL_SECTIONS:
continue
color = self.get_config_attribute(name, 'color')
if hasattr(color, 'none_setting'):
color = None
mappings[name] = color
# Store mappings for later use.
self.mappings_color = mappings
def process_module_output(self, outputs):
"""
Process the output for a module and return a json string representing it.
Color processing occurs here.
"""
for output in outputs:
# Color: substitute the config defined color
if 'color' not in output:
# Get the module name from the output.
module_name = '{} {}'.format(
output['name'],
output.get('instance', '').split(' ')[0]).strip()
color = self.mappings_color.get(module_name)
if color:
output['color'] = color
# Create the json string output.
return ','.join([dumps(x) for x in outputs])
def i3bar_stop(self, signum, frame):
self.i3bar_running = False
# i3status should be stopped
self.i3status_thread.suspend_i3status()
self.sleep_modules()
def i3bar_start(self, signum, frame):
self.i3bar_running = True
self.wake_modules()
def sleep_modules(self):
# Put all py3modules to sleep so they stop updating
for module in self.output_modules.values():
if module['type'] == 'py3status':
module['module'].sleep()
def wake_modules(self):
# Wake up all py3modules.
for module in self.output_modules.values():
if module['type'] == 'py3status':
module['module'].wake()
@profile
def run(self):
"""
Main py3status loop, continuously read from i3status and modules
and output it to i3bar for displaying.
"""
# SIGUSR1 forces a refresh of the bar both for py3status and i3status,
# this mimics the USR1 signal handling of i3status (see man i3status)
signal(SIGUSR1, self.sig_handler)
signal(SIGTERM, self.terminate)
# initialize usage variables
i3status_thread = self.i3status_thread
py3_config = self.config['py3_config']
# prepare the color mappings
self.create_mappings(py3_config)
# self.output_modules needs to have been created before modules are
# started. This is so that modules can do things like register their
# content_function.
self.create_output_modules()
# Some modules need to be prepared before they can run
# eg run their post_config_hook
for module in self.modules.values():
module.prepare_module()
# modules can now receive updates
self.py3_modules_initialized = True
# start modules
for module in self.modules.values():
module.start_module()
# this will be our output set to the correct length for the number of
# items in the bar
output = [None] * len(py3_config['order'])
interval = self.config['interval']
last_sec = 0
# start our output
header = {'version': 1, 'click_events': True, 'stop_signal': SIGTSTP}
print_line(dumps(header))
print_line('[[]')
update_due = None
# main loop
while True:
# process the timeout_queue and get interval till next update due
update_due = self.timeout_queue_process()
# wait until an update is requested
if self.update_request.wait(timeout=update_due):
# event was set so clear it
self.update_request.clear()
while not self.i3bar_running:
time.sleep(0.1)
sec = int(time.time())
# only check everything is good each second
if sec > last_sec:
last_sec = sec
# check i3status thread
if not i3status_thread.is_alive():
err = i3status_thread.error
if not err:
err = 'I3status died horribly.'
self.notify_user(err)
# check events thread
if not self.events_thread.is_alive():
# don't spam the user with i3-nagbar warnings
if not hasattr(self.events_thread, 'nagged'):
self.events_thread.nagged = True
err = 'Events thread died, click events are disabled.'
self.notify_user(err, level='warning')
# update i3status time/tztime items
if interval == 0 or sec % interval == 0:
update_due = i3status_thread.update_times()
# check if an update is needed
if self.update_queue:
while (len(self.update_queue)):
module_name = self.update_queue.popleft()
module = self.output_modules[module_name]
for index in module['position']:
# store the output as json
out = module['module'].get_latest()
output[index] = self.process_module_output(out)
# build output string
out = ','.join([x for x in output if x])
# dump the line to stdout
print_line(',[{}]'.format(out))
def handle_cli_command(self, config):
"""Handle a command from the CLI.
"""
cmd = config['cli_command']
# aliases
if cmd[0] in ['mod', 'module', 'modules']:
cmd[0] = 'modules'
# allowed cli commands
if cmd[:2] in (['modules', 'list'], ['modules', 'details']):
docstrings.show_modules(config, cmd[1:])
# docstring formatting and checking
elif cmd[:2] in (['docstring', 'check'], ['docstring', 'update']):
if cmd[1] == 'check':
show_diff = len(cmd) > 2 and cmd[2] == 'diff'
if show_diff:
mods = cmd[3:]
else:
mods = cmd[2:]
docstrings.check_docstrings(show_diff, config, mods)
if cmd[1] == 'update':
if len(cmd) < 3:
print_stderr('Error: you must specify what to update')
sys.exit(1)
if cmd[2] == 'modules':
docstrings.update_docstrings()
else:
docstrings.update_readme_for_modules(cmd[2:])
elif cmd[:2] in (['modules', 'enable'], ['modules', 'disable']):
# TODO: to be implemented
pass
else:
print_stderr('Error: unknown command')
sys.exit(1)
class StreamController:
def __init__(self,
name,
predictor,
stream_in,
stream_out,
anomaly_stream=None,
learning_stream=None,
learning_threshold=100):
self.name = name
self.predictor = predictor
self.stream_in = stream_in
self.stream_out = stream_out
self.anomaly_stream = anomaly_stream
self.learning_stream = learning_stream
self.learning_threshold = learning_threshold
self.learning_data = []
self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
self.stop_event = Event()
self.model_interface = ModelInterfaceWrapper(ModelInterface())
self.data_store = DataStore()
self.config = Config()
p = db.session.query(db.Predictor).filter_by(
company_id=self.company_id, name=self.predictor).first()
if p is None:
raise Exception(f'Predictor {predictor} doesn\'t exist')
self.target = p.to_predict[0]
ts_settings = p.learn_args.get('timeseries_settings', None)
if not ts_settings['is_timeseries']:
ts_settings = None
if ts_settings is None:
self.thread = Thread(target=StreamController._make_predictions,
args=(self, ))
else:
self.ts_settings = ts_settings
self.thread = Thread(target=StreamController._make_ts_predictions,
args=(self, ))
self.thread.start()
def _is_anomaly(self, res):
for k in res:
if k.endswith('_anomaly') and res[k] is not None:
return True
return False
def _consider_learning(self):
if self.learning_stream is not None:
self.learning_data.extend(self.learning_stream.read())
if len(self.learning_data) >= self.learning_threshold:
p = db.session.query(db.Predictor).filter_by(
company_id=self.company_id, name=self.predictor).first()
ds_record = db.session.query(
db.Datasource).filter_by(id=p.datasource_id).first()
df = pd.DataFrame.from_records(self.learning_data)
name = 'name_' + str(time()).replace('.', '_')
path = os.path.join(self.config['paths']['datasources'], name)
df.to_csv(path)
from_data = {
'class': 'FileDS',
'args': [path],
'kwargs': {},
}
self.data_store.save_datasource(name=name,
source_type='file',
source=path,
file_path=path,
company_id=self.company_id)
ds = self.data_store.get_datasource(name, self.company_id)
self.model_interface.adjust(p.name, from_data, ds['id'],
self.company_id)
self.learning_data.clear()
def _make_predictions(self):
while not self.stop_event.wait(0.5):
self._consider_learning()
for when_data in self.stream_in.read():
preds = self.model_interface.predict(self.predictor, when_data,
'dict')
for res in preds:
if self.anomaly_stream is not None and self._is_anomaly(
res):
self.anomaly_stream.write(res)
else:
self.stream_out.write(res)
def _make_ts_predictions(self):
window = self.ts_settings['window']
order_by = self.ts_settings['order_by']
order_by = [order_by] if isinstance(order_by, str) else order_by
group_by = self.ts_settings.get('group_by', None)
group_by = [group_by] if isinstance(group_by, str) else group_by
cache = Cache(self.name)
while not self.stop_event.wait(0.5):
self._consider_learning()
for when_data in self.stream_in.read():
for ob in order_by:
if ob not in when_data:
raise Exception(
f'when_data doesn\'t contain order_by[{ob}]')
for gb in group_by:
if gb not in when_data:
raise Exception(
f'when_data doesn\'t contain group_by[{gb}]')
gb_value = tuple(
when_data[gb]
for gb in group_by) if group_by is not None else ''
# because cache doesn't work for tuples
# (raises Exception: tuple doesn't have "encode" attribute)
gb_value = str(gb_value)
with cache:
if gb_value not in cache:
cache[gb_value] = []
# do this because shelve-cache doesn't support
# in-place changing
records = cache[gb_value]
records.append(when_data)
cache[gb_value] = records
with cache:
for gb_value in cache.keys():
if len(cache[gb_value]) >= window:
cache[gb_value] = [
*sorted(
cache[gb_value],
# WARNING: assuming wd[ob] is numeric
key=lambda wd: tuple(wd[ob]
for ob in order_by))
]
while len(cache[gb_value]) >= window:
res_list = self.model_interface.predict(
self.predictor, cache[gb_value][:window],
'dict')
if self.anomaly_stream is not None and self._is_anomaly(
res_list[-1]):
self.anomaly_stream.write(res_list[-1])
else:
self.stream_out.write(res_list[-1])
cache[gb_value] = cache[gb_value][1:]
def auto_detect_worker(self):
Logger.info('RCPAPI: auto_detect_worker starting')
class VersionResult(object):
version_json = None
def on_ver_win(value):
version_result.version_json = value
version_result_event.set()
while self._running.is_set():
self._auto_detect_event.wait()
self._auto_detect_event.clear()
self._enable_autodetect.wait()
# check again if we're shutting down
# to prevent a needless re-detection attempt
if not self._running.is_set():
break
try:
Logger.debug("RCPAPI: Starting auto-detect")
self._auto_detect_busy.set()
self.sendCommandLock.acquire()
self.addListener("ver", on_ver_win)
comms = self.comms
if comms and comms.isOpen():
comms.close()
version_result = VersionResult()
version_result_event = Event()
version_result_event.clear()
if comms.device:
devices = [comms.device]
else:
devices = comms.get_available_devices()
last_known_device = self._settings.userPrefs.get_pref(
'preferences', 'last_known_device')
# if there was a last known device try this one first.
if last_known_device:
Logger.info(
'RCPAPI: trying last known device first: {}'.
format(last_known_device))
# ensure we remove it from the existing list
try:
devices.remove(last_known_device)
except ValueError:
pass
devices = [last_known_device] + devices
Logger.debug('RCPAPI: Searching for device')
testVer = VersionConfig()
for device in devices:
try:
Logger.debug('RCPAPI: Trying ' + str(device))
if self.detect_activity_callback:
self.detect_activity_callback(str(device))
comms.device = device
comms.open()
self.sendGetVersion()
version_result_event.wait(2)
version_result_event.clear()
if version_result.version_json != None:
testVer.fromJson(
version_result.version_json.get('ver', None))
if testVer.is_valid:
break # we found something!
else:
try:
Logger.debug('RCPAPI: Giving up on ' +
str(device))
comms.close()
finally:
pass
except Exception as detail:
Logger.error('RCPAPI: Not found on ' + str(device) +
" " + str(detail))
Logger.error(traceback.format_exc())
try:
comms.close()
finally:
pass
if testVer.is_valid:
Logger.debug("RCPAPI: Found device version " +
str(testVer) + " on port: " +
str(comms.device))
self.detect_win(testVer)
self._auto_detect_event.clear()
self._settings.userPrefs.set_pref('preferences',
'last_known_device',
comms.device)
else:
Logger.debug('RCPAPI: Did not find device')
comms.close()
comms.device = None
if self.detect_fail_callback: self.detect_fail_callback()
except Exception as e:
Logger.error('RCPAPI: Error running auto detect: ' + str(e))
Logger.error(traceback.format_exc())
if self.detect_fail_callback: self.detect_fail_callback()
finally:
Logger.debug("RCPAPI: auto detect finished. port=" +
str(comms.device))
self._auto_detect_busy.clear()
self.removeListener("ver", on_ver_win)
self.sendCommandLock.release()
comms.device = None
sleep(AUTODETECT_COOLOFF_TIME)
safe_thread_exit()
Logger.debug('RCPAPI: auto_detect_worker exiting')
class LogProcessingWorker(Thread): # pylint: disable=too-many-instance-attributes
""""""
# ----------------------------------------------------------------------
def __init__(self, *args, **kwargs):
self._host = kwargs.pop('host')
self._port = kwargs.pop('port')
self._transport = kwargs.pop('transport')
self._ssl_enable = kwargs.pop('ssl_enable')
self._memory_cache = kwargs.pop('cache')
self._event_ttl = kwargs.pop('event_ttl')
super().__init__(*args, **kwargs)
self.daemon = True
self.name = self.__class__.__name__
self._shutdown_event = Event()
self._flush_event = Event()
self._queue = Queue()
self._event = None
self._last_event_flush_date = None
self._non_flushed_event_count = None
self._logger = None
self._rate_limit_storage = None
self._rate_limit_strategy = None
self._rate_limit_item = None
# ----------------------------------------------------------------------
def enqueue_event(self, event):
# called from other threads
self._queue.put(event)
# ----------------------------------------------------------------------
def shutdown(self):
# called from other threads
self._shutdown_event.set()
# ----------------------------------------------------------------------
def run(self):
self._reset_flush_counters()
self._setup_logger()
self._setup_memory_cache()
try:
self._fetch_events()
except Exception as exc:
# we really should not get anything here, and if, the worker thread is dying
# too early resulting in undefined application behaviour
self._log_general_error(exc)
# check for empty queue and report if not
self._warn_about_non_empty_queue_on_shutdown()
# ----------------------------------------------------------------------
def force_flush_queued_events(self):
self._flush_event.set()
# ----------------------------------------------------------------------
def _reset_flush_counters(self):
self._last_event_flush_date = datetime.now()
self._non_flushed_event_count = 0
# ----------------------------------------------------------------------
def _clear_flush_event(self):
self._flush_event.clear()
# ----------------------------------------------------------------------
def _setup_logger(self):
self._logger = get_logger(self.name)
# rate limit our own messages to not spam around in case of temporary network errors, etc
rate_limit_setting = constants.ERROR_LOG_RATE_LIMIT
if rate_limit_setting:
self._rate_limit_storage = MemoryStorage()
self._rate_limit_strategy = FixedWindowRateLimiter(
self._rate_limit_storage)
self._rate_limit_item = parse_rate_limit(rate_limit_setting)
# ----------------------------------------------------------------------
def _setup_memory_cache(self):
self._memory_cache = MemoryCache(cache=self._memory_cache,
event_ttl=self._event_ttl)
# ----------------------------------------------------------------------
def _fetch_events(self):
while True:
try:
self._fetch_event()
self._process_event()
except Empty:
# Flush queued (in database) events after internally queued events has been
# processed, i.e. the queue is empty.
if self._shutdown_requested():
self._flush_queued_events(force=True)
return
force_flush = self._flush_requested()
self._flush_queued_events(force=force_flush)
self._delay_processing()
self._expire_events()
except ProcessingError:
if self._shutdown_requested():
return
self._requeue_event()
self._delay_processing()
# ----------------------------------------------------------------------
def _fetch_event(self):
self._event = self._queue.get(block=False)
# ----------------------------------------------------------------------
def _process_event(self):
try:
self._write_event_to_database()
except Exception as exc:
self._log_processing_error(exc)
raise ProcessingError from exc
else:
self._event = None
# ----------------------------------------------------------------------
def _expire_events(self):
self._memory_cache.expire_events()
# ----------------------------------------------------------------------
def _log_processing_error(self, exception):
self._safe_log(u'exception',
u'Log processing error (queue size: %3s): %s',
self._queue.qsize(),
exception,
exc=exception)
# ----------------------------------------------------------------------
def _delay_processing(self):
self._shutdown_event.wait(constants.QUEUE_CHECK_INTERVAL)
# ----------------------------------------------------------------------
def _shutdown_requested(self):
return self._shutdown_event.is_set()
# ----------------------------------------------------------------------
def _flush_requested(self):
return self._flush_event.is_set()
# ----------------------------------------------------------------------
def _requeue_event(self):
self._queue.put(self._event)
# ----------------------------------------------------------------------
def _write_event_to_database(self):
self._memory_cache.add_event(self._event)
self._non_flushed_event_count += 1
# ----------------------------------------------------------------------
def _flush_queued_events(self, force=False):
# check if necessary and abort if not
if not force and not self._queued_event_interval_reached() and \
not self._queued_event_count_reached():
return
self._clear_flush_event()
while True:
queued_events = self._fetch_queued_events_for_flush()
if not queued_events:
break
try:
events = [event['event_text'] for event in queued_events]
self._send_events(events)
# exception types for which we do not want a stack trace
except (ConnectionError, TimeoutError, socket_gaierror) as exc:
self._safe_log(u'error',
u'An error occurred while sending events: %s',
exc)
self._memory_cache.requeue_queued_events(queued_events)
break
except Exception as exc:
self._safe_log(u'exception',
u'An error occurred while sending events: %s',
exc,
exc=exc)
self._memory_cache.requeue_queued_events(queued_events)
break
else:
self._delete_queued_events_from_database()
self._reset_flush_counters()
# ----------------------------------------------------------------------
def _fetch_queued_events_for_flush(self):
try:
return self._memory_cache.get_queued_events()
except Exception as exc:
# just log the exception and hope we can recover from the error
self._safe_log(u'exception',
u'Error retrieving queued events: %s',
exc,
exc=exc)
return None
# ----------------------------------------------------------------------
def _delete_queued_events_from_database(self):
self._memory_cache.delete_queued_events()
# ----------------------------------------------------------------------
def _queued_event_interval_reached(self):
delta = datetime.now() - self._last_event_flush_date
return delta.total_seconds() > constants.QUEUED_EVENTS_FLUSH_INTERVAL
# ----------------------------------------------------------------------
def _queued_event_count_reached(self):
return self._non_flushed_event_count > constants.QUEUED_EVENTS_FLUSH_COUNT
# ----------------------------------------------------------------------
def _send_events(self, events):
use_logging = not self._shutdown_requested()
self._transport.send(events, use_logging=use_logging)
# ----------------------------------------------------------------------
def _log_general_error(self, exc):
self._safe_log(u'exception',
u'An unexpected error occurred: %s',
exc,
exc=exc)
# ----------------------------------------------------------------------
def _safe_log(self, log_level, message, *args, **kwargs):
# we cannot log via the logging subsystem any longer once it has been set to shutdown
if self._shutdown_requested():
safe_log_via_print(log_level, message, *args, **kwargs)
else:
rate_limit_allowed = self._rate_limit_check(kwargs)
if rate_limit_allowed <= 0:
return # skip further logging due to rate limiting
if rate_limit_allowed == 1:
# extend the message to indicate future rate limiting
message = \
u'{} (rate limiting effective, ' \
'further equal messages will be limited)'.format(message)
self._safe_log_impl(log_level, message, *args, **kwargs)
# ----------------------------------------------------------------------
def _rate_limit_check(self, kwargs):
exc = kwargs.pop('exc', None)
if self._rate_limit_strategy is not None and exc is not None:
key = self._factor_rate_limit_key(exc)
# query curent counter for the caller
_, remaining = self._rate_limit_strategy.get_window_stats(
self._rate_limit_item, key)
# increase the rate limit counter for the key
self._rate_limit_strategy.hit(self._rate_limit_item, key)
return remaining
return 2 # any value greater than 1 means allowed
# ----------------------------------------------------------------------
def _factor_rate_limit_key(self, exc): # pylint: disable=no-self-use
module_name = getattr(exc, '__module__', '__no_module__')
class_name = exc.__class__.__name__
key_items = [module_name, class_name]
if hasattr(exc, 'errno') and isinstance(exc.errno, int):
# in case of socket.error, include the errno as rate limiting key
key_items.append(str(exc.errno))
return '.'.join(key_items)
# ----------------------------------------------------------------------
def _safe_log_impl(self, log_level, message, *args, **kwargs):
log_func = getattr(self._logger, log_level)
log_func(message, *args, **kwargs)
# ----------------------------------------------------------------------
def _warn_about_non_empty_queue_on_shutdown(self):
queue_size = self._queue.qsize()
if queue_size:
self._safe_log(
'warn',
u'Non-empty queue while shutting down ({} events pending). '
u'This indicates a previous error.'.format(queue_size),
extra=dict(queue_size=queue_size))
class BlueDot(Dot):
"""
Interacts with a Blue Dot client application, communicating when and where a
button has been pressed, released or held.
This class starts an instance of :class:`.btcomm.BluetoothServer`
which manages the connection with the Blue Dot client.
This class is intended for use with a Blue Dot client application.
The following example will print a message when the Blue Dot button is pressed::
from bluedot import BlueDot
bd = BlueDot()
bd.wait_for_press()
print("The button was pressed")
Multiple buttons can be created, by changing the number of columns and rows. Each button can be referenced using its [col, row]::
bd = BlueDot(cols=2, rows=2)
bd[0,0].wait_for_press()
print("Top left button pressed")
bd[1,1].wait_for_press()
print("Bottom right button pressed")
:param str device:
The Bluetooth device the server should use, the default is "hci0", if
your device only has 1 Bluetooth adapter this shouldn't need to be changed.
:param int port:
The Bluetooth port the server should use, the default is 1, and under
normal use this should never need to change.
:param bool auto_start_server:
If ``True`` (the default), the Bluetooth server will be automatically
started on initialisation; if ``False``, the method :meth:`start` will
need to be called before connections will be accepted.
:param bool power_up_device:
If ``True``, the Bluetooth device will be powered up (if required) when the
server starts. The default is ``False``.
Depending on how Bluetooth has been powered down, you may need to use :command:`rfkill`
to unblock Bluetooth to give permission to bluez to power on Bluetooth::
sudo rfkill unblock bluetooth
:param bool print_messages:
If ``True`` (the default), server status messages will be printed stating
when the server has started and when clients connect / disconnect.
:param int cols:
The number of columns in the grid of buttons. Defaults to ``1``.
:param int rows:
The number of rows in the grid of buttons. Defaults to ``1``.
"""
def __init__(self,
device="hci0",
port=1,
auto_start_server=True,
power_up_device=False,
print_messages=True,
cols=1,
rows=1,
command_callback=None):
self._data_buffer = ""
self._device = device
self._port = port
self._power_up_device = power_up_device
self._print_messages = print_messages
self._check_protocol_event = Event()
self._is_connected_event = Event()
self._when_client_connects = None
self._when_client_connects_background = False
self._when_client_disconnects = None
self._when_client_disconnects_background = False
# setup command callback to externally defined function
self._is_command = command_callback
# setup the main "dot"
super().__init__(BLUE, False, False, True)
# setup the grid
self._buttons = {}
self.resize(cols, rows)
self._create_server()
if auto_start_server:
self.start()
@property
def buttons(self):
"""
A list of :class:`BlueDotButton` objects in the "grid".
"""
return self._buttons.values()
@property
def cols(self):
"""
Sets or returns the number of columns in the grid of buttons.
"""
return self._cols
@cols.setter
def cols(self, value):
self.resize(value, self._rows)
@property
def rows(self):
"""
Sets or returns the number of rows in the grid of buttons.
"""
return self._rows
@rows.setter
def rows(self, value):
self.resize(self._cols, value)
@property
def device(self):
"""
The Bluetooth device the server is using. This defaults to "hci0".
"""
return self._device
@property
def port(self):
"""
The port the server is using. This defaults to 1.
"""
return self._port
@property
def server(self):
"""
The :class:`.btcomm.BluetoothServer` instance that is being used to communicate
with clients.
"""
return self._server
@property
def adapter(self):
"""
The :class:`.btcomm.BluetoothAdapter` instance that is being used.
"""
return self._server.adapter
@property
def paired_devices(self):
"""
Returns a sequence of devices paired with this adapter
:code:`[(mac_address, name), (mac_address, name), ...]`::
bd = BlueDot()
devices = bd.paired_devices
for d in devices:
device_address = d[0]
device_name = d[1]
"""
return self._server.adapter.paired_devices
@property
def print_messages(self):
"""
When set to ``True`` messages relating to the status of the Bluetooth server
will be printed.
"""
return self._print_messages
@print_messages.setter
def print_messages(self, value):
self._print_messages = value
@property
def running(self):
"""
Returns a ``True`` if the server is running.
"""
return self._server.running
@property
def is_connected(self):
"""
Returns ``True`` if a Blue Dot client is connected.
"""
return self._is_connected_event.is_set()
@property
def is_pressed(self):
"""
Returns ``True`` if the button is pressed (or held).
.. note::
If there are multiple buttons, if any button is pressed, `True`
will be returned.
"""
for button in self.buttons:
if button._is_pressed:
return True
return False
@property
def interaction(self):
"""
Returns an instance of :class:`BlueDotInteraction` representing the
current or last interaction with the Blue Dot.
.. note::
If the Blue Dot is released (and inactive), :attr:`interaction`
will return the interaction when it was released, until it is
pressed again. If the Blue Dot has never been pressed
:attr:`interaction` will return ``None``.
If there are multiple buttons, the interaction will only be
returned for button [0,0]
.. deprecated:: 2.0.0
"""
return self._get_button((0, 0)).interaction
@property
def rotation_segments(self):
"""
Sets or returns the number of virtual segments the button is split into for rotating.
Defaults to 8.
.. note::
If there are multiple buttons in the grid, the 'default' value
will be returned and when set all buttons will be updated.
"""
return super(BlueDot, self.__class__).rotation_segments.fget(self)
@rotation_segments.setter
def rotation_segments(self, value):
super(BlueDot, self.__class__).rotation_segments.fset(self, value)
for button in self.buttons:
button.rotation_segments = value
@property
def double_press_time(self):
"""
Sets or returns the time threshold in seconds for a double press. Defaults to 0.3.
.. note::
If there are multiple buttons in the grid, the 'default' value
will be returned and when set all buttons will be updated.
"""
return super(BlueDot, self.__class__).double_press_time.fget(self)
@double_press_time.setter
def double_press_time(self, value):
super(BlueDot, self.__class__).double_press_time.fset(self, value)
for button in self.buttons:
button.double_press_time = value
@property
def color(self):
"""
Sets or returns the color of the button. Defaults to BLUE.
An instance of :class:`.colors.Color` is returned.
Value can be set as a :class:`.colors.Color` object, a hex color value
in the format `#rrggbb` or `#rrggbbaa`, a tuple of `(red, green, blue)`
or `(red, green, blue, alpha)` values between `0` & `255` or a text
description of the color, e.g. "red".
A dictionary of available colors can be obtained from `bluedot.COLORS`.
.. note::
If there are multiple buttons in the grid, the 'default' value
will be returned and when set all buttons will be updated.
"""
return super(BlueDot, self.__class__).color.fget(self)
@color.setter
def color(self, value):
super(BlueDot, self.__class__).color.fset(self, value)
for button in self.buttons:
button.color = value
@property
def square(self):
"""
When set to `True` the 'dot' is made square. Default is `False`.
.. note::
If there are multiple buttons in the grid, the 'default' value
will be returned and when set all buttons will be updated.
"""
return super(BlueDot, self.__class__).square.fget(self)
@square.setter
def square(self, value):
super(BlueDot, self.__class__).square.fset(self, value)
for button in self.buttons:
button.square = value
@property
def border(self):
"""
When set to `True` adds a border to the dot. Default is `False`.
.. note::
If there are multiple buttons in the grid, the 'default' value
will be returned and when set all buttons will be updated.
"""
return super(BlueDot, self.__class__).border.fget(self)
@border.setter
def border(self, value):
super(BlueDot, self.__class__).border.fset(self, value)
for button in self.buttons:
button.border = value
@property
def visible(self):
"""
When set to `False` the dot will be hidden. Default is `True`.
.. note::
Events (press, release, moved) are still sent from the dot
when it is not visible.
If there are multiple buttons in the grid, the 'default' value
will be returned and when set all buttons will be updated.
"""
return super(BlueDot, self.__class__).visible.fget(self)
@visible.setter
def visible(self, value):
super(BlueDot, self.__class__).visible.fset(self, value)
for button in self.buttons:
button.visible = value
@property
def when_client_connects(self):
"""
Sets or returns the function which is called when a Blue Dot
application connects.
The function will be run in the same thread and block, to run in a separate
thread use `set_when_client_connects(function, background=True)`
"""
return self._when_client_connects
@when_client_connects.setter
def when_client_connects(self, value):
self.set_when_client_connects(value)
def set_when_client_connects(self, callback, background=False):
"""
Sets the function which is called when a Blue Dot connects.
:param Callable callback:
The function to call, setting to `None` will stop the callback.
:param bool background:
If set to `True` the function will be run in a separate thread
and it will return immediately. The default is `False`.
"""
self._when_client_connects = callback
self._when_client_connects_background = background
@property
def when_client_disconnects(self):
"""
Sets or returns the function which is called when a Blue Dot disconnects.
The function will be run in the same thread and block, to run in a separate
thread use `set_when_client_disconnects(function, background=True)`
"""
return self._when_client_disconnects
@when_client_disconnects.setter
def when_client_disconnects(self, value):
self.set_when_client_disconnects(value)
def set_when_client_disconnects(self, callback, background=False):
"""
Sets the function which is called when a Blue Dot disconnects.
:param Callable callback:
The function to call, setting to `None` will stop the callback.
:param bool background:
If set to `True` the function will be run in a separate thread
and it will return immediately. The default is `False`.
"""
self._when_client_disconnects = callback
self._when_client_disconnects_background = background
def wait_for_connection(self, timeout=None):
"""
Waits until a Blue Dot client connects.
Returns ``True`` if a client connects.
:param float timeout:
Number of seconds to wait for a wait connections, if ``None`` (the default),
it will wait indefinetly for a connection from a Blue Dot client.
"""
return self._is_connected_event.wait(timeout)
def start(self):
"""
Start the :class:`.btcomm.BluetoothServer` if it is not already
running. By default the server is started at initialisation.
"""
self._server.start()
self._print_message("Server started {}".format(
self.server.server_address))
self._print_message("Waiting for connection")
def _create_server(self):
self._server = BluetoothServer(
self._data_received,
when_client_connects=self._client_connected,
when_client_disconnects=self._client_disconnected,
device=self.device,
port=self.port,
power_up_device=self._power_up_device,
auto_start=False)
def stop(self):
"""
Stop the Bluetooth server.
"""
self._server.stop()
def allow_pairing(self, timeout=60):
"""
Allow a Bluetooth device to pair with your Raspberry Pi by putting
the adapter into discoverable and pairable mode.
:param int timeout:
The time in seconds the adapter will remain pairable. If set to ``None``
the device will be discoverable and pairable indefinetly.
"""
self.server.adapter.allow_pairing(timeout=timeout)
def resize(self, cols, rows):
"""
Resizes the grid of buttons.
:param int cols:
The number of columns in the grid of buttons.
:param int rows:
The number of rows in the grid of buttons.
.. note::
Existing buttons will retain their state (color, border, etc) when
resized. New buttons will be created with the default values set
by the :class:`BlueDot`.
"""
self._cols = cols
self._rows = rows
# create new buttons
new_buttons = {}
for c in range(cols):
for r in range(rows):
# if button already exist, reuse it
if (c, r) in self._buttons.keys():
new_buttons[c, r] = self._buttons[c, r]
else:
new_buttons[c,
r] = BlueDotButton(self, c, r, self._color,
self._square, self._border,
self._visible)
self._buttons = new_buttons
self._send_bluedot_config()
def _get_button(self, key):
try:
return self._buttons[key]
except KeyError:
raise ButtonDoesNotExist(
"The button `{}` does not exist".format(key))
def _client_connected(self):
self._is_connected_event.set()
self._print_message("Client connected {}".format(
self.server.client_address))
self._send_bluedot_config()
if self.when_client_connects:
self._process_callback(self.when_client_connects, None,
self._when_client_connects_background)
# wait for the protocol version to be checked.
if not self._check_protocol_event.wait(CHECK_PROTOCOL_TIMEOUT):
self._print_message(
"Protocol version not received from client - do you need to update the client to the latest version?"
)
self._server.disconnect_client()
def _client_disconnected(self):
self._is_connected_event.clear()
self._check_protocol_event.clear()
self._print_message("Client disconnected")
if self.when_client_disconnects:
self._process_callback(self.when_client_disconnects, None,
self._when_client_disconnects_background)
def _data_received(self, data):
#add the data received to the buffer
self._data_buffer += data
#get any full commands ended by \n
last_command = self._data_buffer.rfind("\n")
if last_command != -1:
commands = self._data_buffer[:last_command].split("\n")
#remove the processed commands from the buffer
self._data_buffer = self._data_buffer[last_command + 1:]
self._process_commands(commands)
def _process_commands(self, commands):
for command in commands:
# debug - print each command
# print(command)
# message received is command for Pi and callback is defined
if (command.split(":")[0] == "CMD"
and self._is_command is not None):
self._is_command(command.split(":")[1].strip())
else:
operation = command.split(",")[0]
params = command.split(",")[1:]
# dot change operation?
if operation in ["0", "1", "2"]:
position = None
try:
button, position = self._parse_interaction_msg(
operation, params)
self._position = position
except ValueError:
# warn about the occasional corrupt command
warnings.warn(
"Data received which could not be parsed.\n{}".
format(command))
except ButtonDoesNotExist:
# data received for a button which could not be found
warnings.warn(
"Data received for a button which does not exist.\n{}"
.format(command))
else:
# dot released
if operation == "0":
self._process_release(button, position)
# dot pressed
elif operation == "1":
self._process_press(button, position)
# dot pressed position moved
elif operation == "2":
self._process_move(button, position)
# protocol check
elif operation == "3":
self._check_protocol_version(params[0], params[1])
else:
# operation not identified...
warnings.warn(
"Data received for an unknown operation.\n{}".format(
command))
def _parse_interaction_msg(self, operation, params):
"""
Parses an interaction (press, move, release) message and returns
the component parts
"""
# parse message
col = int(params[0])
row = int(params[1])
position = BlueDotPosition(col, row, params[2], params[3])
button = self._get_button((col, row))
return button, position
def _process_press(self, button, position):
# was the button double pressed?
if button.is_double_press(position):
self.double_press(position)
button.double_press(position)
# set the blue dot and button as pressed
self.press(position)
button.press(position)
def _process_move(self, button, position):
# set the blue dot as moved
self.move(position)
# set the button as moved
button.move(position)
# was it a rotation
rotation = button.get_rotation()
if rotation is not None:
self.rotate(rotation)
button.rotate(rotation)
def _process_release(self, button, position):
# set the blue dot as released
self.release(position)
# set the button as released
button.release(position)
# was it a swipe?
swipe = button.get_swipe()
if swipe is not None:
self.swipe(swipe)
button.swipe(swipe)
def _check_protocol_version(self, protocol_version, client_name):
try:
version_no = int(protocol_version)
except ValueError:
raise ValueError(
"protocol version number must be numeric, received {}.".format(
protocol_version))
self._check_protocol_event.set()
if version_no != PROTOCOL_VERSION:
msg = "Client '{}' was using protocol version {}, bluedot python library is using version {}. "
if version_no > PROTOCOL_VERSION:
msg += "Update the bluedot python library, using 'sudo pip3 --upgrade install bluedot'."
msg = msg.format(client_name, protocol_version,
PROTOCOL_VERSION)
else:
msg += "Update the {}."
msg = msg.format(client_name, protocol_version,
PROTOCOL_VERSION, client_name)
self._server.disconnect_client()
print(msg)
# called whenever the BlueDot configuration is changed or a client connects
def _send_bluedot_config(self):
if self.is_connected:
self._server.send("4,{},{},{},{},{},{}\n".format(
self._color.str_rgba, int(self._square), int(self._border),
int(self._visible), self._cols, self._rows))
# send the configuration for the individual buttons
button_config_msg = ""
for button in self.buttons:
if button.modified:
button_config_msg += button._build_config_msg()
if button_config_msg != "":
self._server.send(button_config_msg)
def _print_message(self, message):
if self.print_messages:
print(message)
def __getitem__(self, key):
return self._get_button(key)
class IBClient(object):
"""IB Socket client"""
def __init__(self,
host='localhost',
port=7496,
client_id=0,
client_name='IB'):
"""
Args:
host: TWS 所在机器的 IP或域名
port: TWS配置的接收外部API的端口.TWS default value: 7496; TWS demo account default value: 7497
client_id: API<->TWS之间 sock连接的ID
client_name: 本次连接的名字。可选
"""
self.client_name = client_name
self.host = host # host IP address in a string; e.g. '127.0.0.1', 'localhost'
self.port = port # socket port;
self.client_id = client_id # socket client id
self.tickerId = 0 # known as ticker ID or request ID
self.ipc_msg_dict = {
} # key: ticker ID or request ID; value: request and response objects; response objects ususally carrys data, Events, and Status
self.order_id = 0 # current available order ID
self.order_history = {
} # key: ticker ID or request ID; value: request and response objects; response objects ususally carrys data, Events, and Status
# dict to store market depth data; key: (client_id, request_id)
self.market_depth_buffer = dict()
self.context = None # key: ticker ID or request ID; value: request and response objects; response objects ususally carrys data, Events, and Status
self.portfolio = None
self.account = None
self.wrapper = IBMsgWrapper(
self) # the instance with IB message callback methods
self.connection = EClientSocket(
self.wrapper) # low layer socket client
# TWS's data connection status
self.hmdf_status_dict = {}
for farm in IB_FARM_NAME_LS:
self.hmdf_status_dict[farm] = 'unknown'
# EVENTS
self.conn_down_event = Event() # sock connection event
self.mdf_conn_event = Event() # market data connection event
self.hdf_conn_event = Event() # hist data connection event
self.order_event = Event()
self.account_event = Event()
self.get_order_event = Event()
self.tick_snapshot_req_end = Event()
# LOCKER
self.req_id_locker = threading.Lock()
# Order ID cond
self.order_id_cond = threading.Condition()
# CONSTANT VALUES
self.PRICE_DF_HEADER1 = [
'time', 'open', 'high', 'low', 'close', 'volume'
]
self.PRICE_DF_HEADER2 = [
'symbol', 'time', 'open', 'high', 'low', 'close', 'volume'
]
@property
def connected(self):
return self.connection.m_connected
def connect(self):
""" Connect to socket host, e.g. TWS """
self.connection.eConnect(self.host, self.port, self.client_id)
timeout = 5.
count = 0.
while not self.connected and count < timeout:
count += 0.05
sleep(0.05)
if self.connected:
self.order_id = self.connection.reqIds(-1)
else:
print('failed to connect.')
return self.connected
def close(self):
""" disconnect from IB host """
self.disconnect()
def disconnect(self):
""" disconnect from IB host """
# self.disable_account_info_update()
self.connection.eDisconnect()
def __get_new_request_id(self):
'''' generate a new request ID (ticker ID) in a thread safe way '''
self.req_id_locker.acquire()
self.tickerId += 1
__id = self.tickerId
self.req_id_locker.release()
return __id
def setup_account(self, account_id, starting_cash):
self.portfolio = Portfolio(account_id, starting_cash)
self.account = self.portfolio.account
if self.connected:
# TODO: may need to move this to a thread or at user layer
self.enable_account_info_update()
#
# Tick Data Methods
#
def request_tick_data(self, contract):
""" Subscribe tick data for a specified contract
Args:
contract: a legal IBPY Contract object or a string for U.S. stock only
Returns:
tickerId: the ID of this request. this ID could be used to cancel request later.
tick_data: a reference to the tick data dictionary which will be updated with latest quote.
"""
if isinstance(contract, Contract):
pass
elif isinstance(contract, str):
contract = new_stock_contract(contract)
else:
raise TypeError(
"contract must be a contract object or string (for U.S. stocks only)."
)
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
__id = self.__get_new_request_id()
request = RequestDetails('reqMktData', 'Snapshot', contract)
response = ResponseDetails()
self.ipc_msg_dict[__id] = (request, response)
# False - indicating request live quotes instead of a snapshot
self.connection.reqMktData(__id, contract, '', False)
return __id, self.ipc_msg_dict[__id][1].tick_data
def get_tick_snapshot(self, contract, max_wait_time=5):
""" Get a snapshot with default tick types and corresponding tick data for a given contract
Note: 1) no generic ticks can be specified.
2) only return data fields which have changed within an 11 second interval.
If it is necessary for an API client to receive a certain data field, it is better
to subscribe to market data until that field has been returned and then cancel the market data request.
Known Issues:
1) When called outside of market hours, get_tick_snapshot request could take more than 10 sec
to reach the end (tickSnapshotEnd)
2) Need to check Issue#1 during market hours
Args:
contract: a legal IBPY Contract object or a string for U.S. stock only
Returns:
a copy of tick data dictionary
Raises:
None
"""
if isinstance(contract, Contract):
pass
elif isinstance(contract, str):
contract = new_stock_contract(contract)
else:
raise TypeError(
"contract must be a contract object or string (for U.S. stocks only)."
)
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
__id = self.__get_new_request_id()
request = RequestDetails('reqMktData', 'Snapshot', contract)
response = ResponseDetails()
self.ipc_msg_dict[__id] = (request, response)
# send reqMktData req
# True - indicating request live quotes instead of a snapshot
#
# Important:
# When set it to 'True', each regulatory snapshot made will incur a fee of 0.01 USD to the account.
# This applies to both live and paper accounts.
self.connection.reqMktData(__id, contract, '', True)
response.event.wait(max_wait_time)
if response.event.is_set():
# tickPrice() and tickeSize() may write after tickSnapshotEnd() completed.
sleep(0.5)
snapshot = copy(response.tick_data)
# remove the from dict and free the memory
response.event.clear()
self.ipc_msg_dict.pop(__id)
else:
response.event.clear()
self.ipc_msg_dict.pop(__id)
raise RuntimeError(
'reqMktData (get_tick_snapshot) is timeout. max_wait_time=%d' %
(max_wait_time))
return snapshot
def cancel_tick_request(self, tickerId):
""" Cancel tick data request for a given ticker ID (request ID)
Args:
tickerId: the ticker request to cancel
Returns:
None
Raises:
None
"""
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
# TODO: check if tickerID is in the list
self.connection.cancelMktData(tickerId)
self.ipc_msg_dict.pop(tickerId)
return
def request_realtime_price(self, contract, price_type='TRADES'):
""" Get real-time price/volume for a specific contract, e.g. stocks, futures and option contracts.
IB API support only 5 sec duration between two real-time bar (price) records.
Args:
contract: one IB contract instance
price_type: 'TRADES', 'MIDPOINT', 'BID', 'ASK'
Returns:
tickerId: the request ID; it's also the key to get response msg from ipc_msg_dict
realtime_price: a reference to the real-time price (OCHL) list which will be updated
with latest price (OCHL) record.
Raises:
None
"""
if isinstance(contract, Contract):
pass
elif isinstance(contract, str):
contract = new_stock_contract(contract)
else:
raise TypeError(
"contract must be a contract object or string (for U.S. stocks only)."
)
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
if price_type not in ['TRADES', 'MIDPOINT', 'BID', 'ASK']:
raise TypeError("Got incorrect price_type")
__id = self.__get_new_request_id()
request = RequestDetails('reqHistoricalData', price_type, contract)
response = ResponseDetails()
self.ipc_msg_dict[__id] = (request, response)
# only 5 sec duration supported
# price_type: 'TRADES', 'MIDPOINT', 'BID', 'ASK'
# useRTH - set to True
self.connection.reqRealTimeBars(__id, contract, 5, price_type, True)
return __id, self.ipc_msg_dict[__id][1].rt_price
def cancel_realtime_price(self, req_id):
""" Cancel realtime price/volumne request.
Args:
req_id: the ticker ID (or request ID)
Returns:
None
"""
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
self.connection.cancelRealTimeBars(req_id)
# remove request/response data from ipc_msg_dict
self.ipc_msg_dict.pop(req_id)
return
#
# Historical Data Methods
#
def get_price_history(self,
contract,
ts_end,
duration='1 M',
frequency='daily',
max_wait_time=30):
""" Get price/volumne history for a specific contract, e.g. stocks, futures and option ocntract.
Args:
contract: one IB contract instance
ts_end: a string in '%Y%m%d' or '%Y%m%d %H:%M:%S' format
duration: string
X S Seconds
X D Day
X W Week
X M Month
X Y Year
frequency: {‘daily’, ‘minute’}, optional; Resolution of the data to be returned.
max_wait_time: int; max num of sec to wait after calling reqHistoricalData
Returns:
pandas Panel/DataFrame/Series – The pricing data that was requested.
Open High Low Close Volume
Date
2017-12-15 6.96 6.96 6.86 6.90 366523000
2017-12-18 6.88 7.02 6.87 6.98 303664000
2017-12-19 7.00 7.02 6.98 7.01 299342000
Raises:
None
"""
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
if isinstance(contract, Contract):
pass
elif isinstance(contract, str):
contract = new_stock_contract(contract)
else:
raise TypeError("contract must be a Contract object or string")
if frequency == 'daily':
bar_size = '1 day'
elif frequency == 'minute':
bar_size = '1 min'
elif frequency == 'second':
bar_size = '1 sec'
elif frequency == '5 seconds':
bar_size = '5 secs'
else:
raise ValueError("get_price_history: incorrect frequency value")
if len(ts_end) == 8 or len(ts_end) == 17:
if len(ts_end) == 8:
ts_end = ts_end + ' 23:59:59'
else:
print('get_price_history: incorrect ts_end format')
return
__id = self.__get_new_request_id()
request = RequestDetails('reqHistoricalData', '', contract)
response = ResponseDetails()
self.ipc_msg_dict[__id] = (request, response)
self.connection.reqHistoricalData(tickerId=__id,
contract=contract,
endDateTime=ts_end,
durationStr=duration,
barSizeSetting=bar_size,
whatToShow='TRADES',
useRTH=0,
formatDate=1)
df = None
response.event.wait(max_wait_time)
if response.event.is_set():
df = pd.DataFrame(response.price_hist,
columns=self.PRICE_DF_HEADER1)
# clean up the time format
print(df.head(10))
print(response.price_hist)
date = df['time'][0]
if len(date) == 8:
df['time'] = pd.to_datetime(df['time'], format='%Y%m%d')
elif len(date) == 18:
# len('20161020 23:46:00') --> 2 Spaces!!!!!
# adj_date = datetime.strptime(date, "%Y%m%d %H:%M:%S")
df['time'] = pd.to_datetime(df['time'],
format="%Y%m%d %H:%M:%S")
else:
# adj_date = datetime.strptime(date, "%Y%m%d %H:%M:%S")
df['time'] = pd.to_datetime(df['time'],
format="%Y%m%d %H:%M:%S")
# TODO: check for timezone
# exchange = request.contract.m_exchange
# server_timezone = pytz.timezone("Asia/Shanghai") # timezone where the server runs
# mkt_timezone = pytz.timezone(IBEXCHANGE.get_timezone(exchange)) # Get Exchange's timezone
# adj_date = server_timezone.localize(adj_date).astimezone(
# mkt_timezone) # covert server time to Exchange's time
# adj_date = adj_date.strftime("%Y%m%d %H:%M:%S") # from datetime to string
df = df.set_index('time')
# remove the from dict and free the memory
response.event.clear()
self.ipc_msg_dict.pop(__id)
else:
self.ipc_msg_dict.pop(__id)
print('reqHistoricalData is timeout.')
raise RuntimeError('reqHistoricalData is timeout.')
return df
def get_stock_price_history(self,
security_list,
ts_end,
duration='1 M',
frequency='daily',
max_wait_time=30):
"""Get price/volumne history for a list of stocks.
Args:
security_list: a list of security symbols, .e.g. ['IBM', 'DATA']
endDateTime: a string in '%Y%m%d' or '%Y%m%d %H:%M:%S' format
durationStr: see IB API doc.
frequency: {‘daily’, ‘minute’}, optional; Resolution of the data to be returned.
max_wait_time: int; max num of sec to wait after calling reqHistoricalData
Returns:
pandas Panel/DataFrame/Series – The pricing data that was requested.
Raises:
None
"""
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
num_secs = len(security_list)
if num_secs <= 0:
return
if frequency == 'daily':
bar_size = '1 day'
elif frequency == 'minute':
bar_size = '1 min'
elif frequency == 'second':
bar_size = '1 sec'
elif frequency == '5 seconds':
bar_size = '5 secs'
else:
print('get_stock_price_history: incorrect frequency')
return
if len(ts_end) == 8 or len(ts_end) == 17:
if len(ts_end) == 8:
ts_end = ts_end + ' 23:59:59'
else:
print('get_stock_price_history: incorrect ts_end format')
return
# ['Symbol', 'Date', 'Open', 'High', 'Low', 'Close', 'Volume']
df = pd.DataFrame(columns=self.PRICE_DF_HEADER2)
for sec in security_list:
__id = self.__get_new_request_id()
contract = new_stock_contract(sec)
request = RequestDetails('reqHistoricalData', '', contract)
response = ResponseDetails()
self.ipc_msg_dict[__id] = (request, response)
self.connection.reqHistoricalData(tickerId=__id,
contract=contract,
endDateTime=ts_end,
durationStr=duration,
barSizeSetting=bar_size,
whatToShow='TRADES',
useRTH=0,
formatDate=1)
response.event.wait(max_wait_time)
if response.event.is_set():
df_tmp = pd.DataFrame(response.price_hist,
columns=self.PRICE_DF_HEADER1)
# clean up the time format
date = df_tmp['time'][0]
if len(date) == 8:
df_tmp['time'] = pd.to_datetime(df_tmp['time'],
format='%Y%m%d')
elif len(date) == 18:
# len('20161020 23:46:00') --> 2 Spaces!!!!!
# adj_date = datetime.strptime(date, "%Y%m%d %H:%M:%S")
df_tmp['time'] = pd.to_datetime(df_tmp['time'],
format="%Y%m%d %H:%M:%S")
else:
# adj_date = datetime.strptime(date, "%Y%m%d %H:%M:%S")
df_tmp['time'] = pd.to_datetime(df_tmp['time'],
format="%Y%m%d %H:%M:%S")
# TODO: check for timezone
# exchange = request.contract.m_exchange
# server_timezone = pytz.timezone("Asia/Shanghai") # timezone where the server runs
# mkt_timezone = pytz.timezone(IBEXCHANGE.get_timezone(exchange)) # Get Exchange's timezone
# adj_date = server_timezone.localize(adj_date).astimezone(
# mkt_timezone) # covert server time to Exchange's time
# adj_date = adj_date.strftime("%Y%m%d %H:%M:%S") # from datetime to string
df_tmp['symbol'] = pd.DataFrame([sec] * len(df_tmp))
df = df.append(df_tmp)
# remove the from dict and free the memory
self.ipc_msg_dict.pop(__id)
response.event.clear()
else:
self.ipc_msg_dict.pop(__id)
raise RuntimeError('reqHistoricalData is timeout.')
return df
def get_contract_price_history(self,
contract,
ts_end,
duration='1 M',
frequency='daily',
max_wait_time=30):
# Same function as get_price_history
return self.get_price_history(contract, ts_end, duration, frequency,
max_wait_time)
#
# Placing/Changing/Canceling Order Methods
#
def request_order_id(self):
""" request order id from TWS client """
# request next valid order ID from IB host; this request will update self.order_id
# details: https: // interactivebrokers.github.io / tws - api / order_submission.html
self.connection.reqIds(-1)
self.order_id_cond.acquire()
self.order_id_cond.wait()
new_order_id = self.order_id
self.order_id_cond.release()
return new_order_id
def get_order_status(self, order_id):
"""
orderStatus 1 PreSubmitted 0 1000 0.0 1216371623 0 0.0 8615 None
{'contract': <ib.ext.Contract.Contract object at 0x10ebb7cf8>,
'order': <ib.ext.Order.Order object at 0x10ebb7668>,
'status': 'PreSubmitted', 'filled': False, 'permId': 1216371623,
'remaining': 1000, 'avgFillPrice': 0.0, 'lastFillPrice': 0.0, 'whyHeld': None}
:param order_id:
:return:
"""
key = (self.client_id, order_id)
order_info = self.order_history.get(key, None)
if order_info:
return order_info['status']
else:
return ""
def order_amount(self, contract, amount, style=MarketOrder()):
''' Place an order. Order X units of security Y.
Warning: only mkt order and limited order work; calling stoploss/stoplimited order will result in IB disconnection.
:param contract: A IB Contract object.
:param amount: The integer amount of shares. Positive means buy, negative means sell.
:param style:
:return:
'''
if amount == 0:
return -1
elif amount > 0:
action = 'BUY'
else:
action = 'SELL'
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
if not isinstance(contract, Contract):
raise TypeError("contract must be a contract object")
order = Order()
# request next valid order ID from IB host; this request will update self.order_id
# details: https: // interactivebrokers.github.io / tws - api / order_submission.html
# self.connection.reqIds(-1)
# sleep(0.05)
order.m_orderId = self.request_order_id()
order.m_client_id = self.client_id
order.m_action = action
order.m_totalQuantity = abs(amount)
order.m_orderType = style.order_type
if style.limit_price is not None:
order.m_lmtPrice = style.limit_price
if style.stop_price is not None:
order.m_auxPrice = style.stop_price
order.m_overridePercentageConstraints = True # override TWS order size constraints
# place order
self.connection.placeOrder(order.m_orderId, contract, order)
# self.order_history[(self.order_id, self.client_id)] = order
# TODO: wait for returns from orderStatus
return order.m_orderId
def combo_order_amount(self, contract, amount, style=MarketOrder()):
''' Place an order
:param contract: A security object.
:param amount: The integer amount of shares. Positive means buy, negative means sell.
:param style:
:return:
'''
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
if amount == 0:
return -1
elif amount > 0:
action = 'BUY'
else:
action = 'SELL'
if isinstance(contract, Contract):
if len(contract.m_comboLegs) == 0:
raise TypeError("contract must contains combo legs")
else:
raise TypeError("contract must be a contract object")
# request next valid order ID from IB host; this request will update self.order_id
self.connection.reqIds(-1) # note: input param is always ignored;
sleep(0.05)
order = Order()
order.m_orderId = self.order_id
order.m_client_id = self.client_id
order.m_action = action
order.m_totalQuantity = abs(amount)
order.m_orderType = style.order_type
if style.limit_price is not None:
order.m_lmtPrice = style.limit_price
if style.stop_price is not None:
order.m_auxPrice = style.stop_price
order.m_overridePercentageConstraints = True # override TWS order size constraints
'''
# Advanced configuration. Not tested yet.
if style.is_combo_order:
if style.non_guaranteed:
tag = TagValue()
tag.m_tag = "NonGuaranteed"
tag.m_value = "1"
order.m_smartComboRoutingParams = [tag]
'''
self.connection.placeOrder(self.order_id, contract, order)
return self.order_id
def order_value(self, contract, value, style):
''' Reserve for future implementation
:param contract:
:param value:
:param style:
:return:
'''
pass
def order_target(self, contract, amount, style):
''' Places an order to adjust a position to a target number of shares.
:param contract:
:param value:
:param style:
:return:
'''
pass
def order_target_value(self, contract, amount, style):
''' Places an order to adjust a position to a target value.
:param contract:
:param value:
:param style:
:return:
'''
pass
def modify_order(self, order_id, contract, amount, style=MarketOrder()):
''' Change amount or order type (including limited price for limtied orders)
for a existing order specified by order_id
:param order_id: a existing order's order_id
:param contract: A IB Contract object. supposed to be the same with the order-to-be-modified.
:param amount: The integer amount of shares. Positive means buy, negative means sell.
:param style: market order or limited order
:return: the existing order's order_id (same as the input)
'''
if amount == 0:
return -1
elif amount > 0:
action = 'BUY'
else:
action = 'SELL'
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
if not isinstance(contract, Contract):
raise TypeError("contract must be a contract object")
order = Order()
order.m_orderId = order_id
order.m_client_id = self.client_id
order.m_action = action
order.m_totalQuantity = abs(amount)
order.m_orderType = style.order_type
if style.limit_price is not None:
order.m_lmtPrice = style.limit_price
if style.stop_price is not None:
order.m_auxPrice = style.stop_price
order.m_overridePercentageConstraints = True # override TWS order size constraints
# place order
self.connection.placeOrder(self.order_id, contract, order)
# TODO: wait for returns from orderStatus
return self.order_id
def cancel_order(self, order):
''' Attempts to cancel the specified order. Cancel is attempted asynchronously.
:param order: Can be the order_id as a string or the order object.
:return: None
'''
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
if isinstance(order, int):
order_id = order
elif isinstance(order, Order):
order_id = order.m_orderId
else:
raise TypeError("order must be a order_id (int) or order object")
self.connection.cancelOrder(order_id)
def get_open_orders(self):
''' Attempts to get all open orders.
:param
:return: None
'''
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
if 'reqOpenOrders' not in self.ipc_msg_dict:
request = RequestDetails('reqOpenOrders', '', '')
response = ResponseDetails()
self.ipc_msg_dict['reqOpenOrders'] = (request, response)
else:
response = self.ipc_msg_dict['reqOpenOrders'][1]
# self.connection.reqOpenOrders()
# self.reqAutoOpenOrders(True)
self.connection.reqAllOpenOrders()
max_wait_time = 3.
self.get_order_event.wait(max_wait_time)
if self.get_order_event.is_set():
# snapshot = copy(response.tick_snapshot)
# self.ipc_msg_dict.pop(self.tickerId)
self.get_order_event.clear()
else:
# self.ipc_msg_dict.pop(self.tickerId)
raise RuntimeError('get_open_orders is timeout.')
return
#
# Account Info Methods
#
def enable_account_info_update(self):
''' Turn on auto account update, meaning IB socket host will push account info to IB socket client.
updateAccountTime()
updateAccountValue()
updatePortfolio()
'''
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
# TODO: check self.IB_acct_id before using it
# request IB host (e.g. TWS) push account info to IB client (socket client)
self.connection.reqAccountUpdates(True, self.account.account_id)
return
def disable_account_info_update(self):
''' Turn off auto account update, meaning IB socket host will stop pushing account info
to IB socket client.
'''
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
# TODO: check self.IB_acct_id before using it
# stop IB host (e.g. TWS) to push account info to IB client (socket client)
self.connection.reqAccountUpdates(False, self.account.account_id)
return
#
# Fundamental Data Methods
#
def get_financial_statements(self, symbol, max_wait_time=20):
''' Get a company's financial statements
:param:
symbol: stock symbol string, e.g. 'IBM'; or a IB contract object
:return:
a string of financial statements
'''
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
if isinstance(symbol, Contract):
contract = symbol
elif isinstance(symbol, str):
contract = new_stock_contract(symbol)
else:
raise TypeError(
"contract must be a contract object or string (for U.S. stocks only)."
)
__id = self.__get_new_request_id()
request = RequestDetails('reqFundamentalData', 'ReportsFinStatements',
contract)
response = ResponseDetails()
self.ipc_msg_dict[__id] = (request, response)
self.connection.reqFundamentalData(__id, contract,
'ReportsFinStatements')
response.event.wait(max_wait_time)
raw_xml = None
if response.event.is_set():
if response.status == ResponseDetails.STATUS_FINISHED:
# covert from xml to dest. format
raw_xml = copy(response.fundamental_data)
else:
pass # raise RuntimeError('get_financial_statements: reqFundamentalData got error. Security=%s Reason:%s' % (symbol, response.error_msg))
else:
# Timeout
pass # ('get_financial_statements: reqFundamentalData is timeout. Security=%s' % symbol)
status = response.status
self.ipc_msg_dict.pop(__id)
return status, raw_xml
def get_company_ownership(self, symbol, max_wait_time=60.0 * 5):
''' Get a company's ownership report
:param:
symbol: stock symbol string, e.g. 'IBM'
max_wait_time: max number of seconds to wait before raise timeout
:return:
a string of ownership report
'''
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
if isinstance(symbol, Contract):
contract = symbol
elif isinstance(symbol, str):
# For US stock only
contract = new_stock_contract(symbol)
else:
raise TypeError(
"contract must be a contract object or string (for U.S. stocks only)."
)
__id = self.__get_new_request_id()
request = RequestDetails('reqFundamentalData', 'ReportsOwnership',
contract)
response = ResponseDetails()
self.ipc_msg_dict[__id] = (request, response)
self.connection.reqFundamentalData(__id, contract, 'ReportsOwnership')
response.event.wait(max_wait_time)
report = None
if response.event.is_set():
if response.status == ResponseDetails.STATUS_FINISHED:
# covert from xml to dest. format
report = parse_ownership_report(response.fundamental_data)
else:
pass # ('get_company_ownership: reqFundamentalData got error. Security=%s Reason:%s' % (symbol, response.error_msg))
else:
pass # ('get_company_ownership: reqFundamentalData is timeout. Security=%s' % symbol)
status = response.status
self.ipc_msg_dict.pop(__id)
return status, report
def get_analyst_estimates(self, symbol, max_wait_time=20):
''' Get analyst estimates report for a company
:param:
symbol: stock symbol string, e.g. 'IBM'; or a IB contract object
:return:
a string of financial statements
'''
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
if isinstance(symbol, Contract):
contract = symbol
elif isinstance(symbol, str):
contract = new_stock_contract(symbol)
else:
raise TypeError(
"contract must be a contract object or string (for U.S. stocks only)."
)
__id = self.__get_new_request_id()
request = RequestDetails('reqFundamentalData',
'RESC-Analyst Estimates', contract)
response = ResponseDetails()
self.ipc_msg_dict[__id] = (request, response)
self.connection.reqFundamentalData(__id, contract, 'RESC')
response.event.wait(max_wait_time)
report = None
if response.event.is_set():
if response.status == ResponseDetails.STATUS_FINISHED:
# covert from xml to dest. format
report = parse_analyst_estimates(response.fundamental_data)
else:
pass # ('get_analyst_estimates: reqFundamentalData got error. Security=%s Reason:%s' % (symbol, response.error_msg))
else:
pass # ('get_analyst_estimates: reqFundamentalData is timeout. Security=%s' % symbol)
status = response.status
self.ipc_msg_dict.pop(__id)
return status, report
def get_company_overview(self, symbol, max_wait_time=10):
''' Get company overview infomration
:param:
symbol: stock symbol string, e.g. 'IBM'; or a IB contract object
:return:
a string of financial statements
'''
# ReportsFinSummary Financial summary
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
if isinstance(symbol, Contract):
contract = symbol
elif isinstance(symbol, str):
contract = new_stock_contract(symbol)
else:
raise TypeError(
"contract must be a contract object or string (for U.S. stocks only)."
)
__id = self.__get_new_request_id()
request = RequestDetails('reqFundamentalData',
'ReportSnapshot-Company overview', contract)
response = ResponseDetails()
self.ipc_msg_dict[__id] = (request, response)
# ReportSnapshot Company's financial overview
self.connection.reqFundamentalData(__id, contract, 'ReportSnapshot')
response.event.wait(max_wait_time)
report = None
if response.event.is_set():
if response.status == ResponseDetails.STATUS_FINISHED:
# TODO: covert from xml to dest. format
report = response.fundamental_data
else:
pass # ('get_analyst_estimates: reqFundamentalData got error. Security=%s Reason:%s' % (symbol, response.error_msg))
else:
pass # ('get_analyst_estimates: reqFundamentalData is timeout. Security=%s' % symbol)
status = response.status
self.ipc_msg_dict.pop(__id)
return status, report
def get_financial_summary(self, symbol, max_wait_time=10):
''' Get company finanical summary information, such as revenue history, net profit, and dividends history.
:param:
symbol: stock symbol string, e.g. 'IBM'; or a IB contract object
:return:
a string of financial statements
'''
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
if isinstance(symbol, Contract):
contract = symbol
elif isinstance(symbol, str):
contract = new_stock_contract(symbol)
else:
raise TypeError(
"contract must be a contract object or string (for U.S. stocks only)."
)
__id = self.__get_new_request_id()
request = RequestDetails('reqFundamentalData',
'ReportsFinSummary-Financial summary',
contract)
response = ResponseDetails()
self.ipc_msg_dict[__id] = (request, response)
self.connection.reqFundamentalData(__id, contract, 'ReportsFinSummary')
response.event.wait(max_wait_time)
report = None
if response.event.is_set():
if response.status == ResponseDetails.STATUS_FINISHED:
# TODO: covert from xml to dest. format
report = response.fundamental_data
else:
pass
else:
pass
status = response.status
self.ipc_msg_dict.pop(__id)
return status, report
def get_financial_ratios(self, symbol, max_wait_time=5):
''' Get analyst estimates report for a company
:param:
symbol: stock symbol string, e.g. 'IBM'
:return:
a string of financial statements
'''
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
if isinstance(symbol, Contract):
contract = symbol
elif isinstance(symbol, str):
contract = new_stock_contract(symbol)
else:
raise TypeError(
"contract must be a contract object or string (for U.S. stocks only)."
)
__id = self.__get_new_request_id()
request = RequestDetails('reqFundamentalData',
'RESC-Analyst Estimates', contract)
response = ResponseDetails()
self.ipc_msg_dict[__id] = (request, response)
# 258 - financial ratios
'''
TTMNPMGN=16.1298;NLOW=80.6;TTMPRCFPS=6.26675;TTMGROSMGN=60.76731;TTMCFSHR=15.004
46;QCURRATIO=1.42071;TTMREV=259842;TTMINVTURN=5.28024;TTMOPMGN=14.22711;TTMPR2RE
V=1.39703;AEPSNORM=8.55;TTMNIPEREM=144524.1;EPSCHNGYR=8.47727;TTMPRFCFPS=62.4260
6;TTMRECTURN=19.99938;TTMPTMGN=17.88125;QCSHPS=40.50882;TTMFCF=5815;
LATESTADATE=2016-12-31;APTMGNPCT=17.88125;AEBTNORM=46463;TTMNIAC=33008;NetDebt_I=152080;
PRYTDPCTR=-1.55563;TTMEBITD=53326;AFEEPSNTM=0;PR2TANBK=5.01599;EPSTRENDGR=-
15.53209;QTOTD2EQ=72.60778;TTMFCFSHR=1.50625;QBVPS=110.0867;NPRICE=94.1;YLD5YAVG
=3.88751;REVTRENDGR=51.11774;TTMEPSXCLX=8.54981;QTANBVPS=18.75999;PRICE2BK=0.854
78;MKTCAP=363007.5;TTMPAYRAT=31.32574;TTMINTCOV=-99999.99;TTMDIVSHR=2.585;TTMREVCHG=55.81794;
TTMROAPCT=4.09615;TTMROEPCT=7.73685;
TTMREVPERE=896006.9;APENORM=11.00585;TTMROIPCT=5.51924;REVCHNGYR=-
6.66885;CURRENCY=HKD;DIVGRPCT=-8.33887;TTMEPSCHG=-32.80548;PEEXCLXOR=11.00609;QQUICKRATI=1.30087;
TTMREVPS=67.30638;BETA=0.90979;TTMEBT=46463;ADIV5YAVG=3.1048;ANIACNORM=33008;QLTD2EQ=55.46377;NHIG=103.9
'''
report = None
self.connection.reqMktData(__id, contract, "258", False)
response.event.wait(max_wait_time)
if response.event.is_set():
if response.status == ResponseDetails.STATUS_FINISHED:
# TODO: convert the format to a table alike
report = response.tick_str
return report
def get_dividends_info(self, symbol, max_wait_time=5):
''' Get analyst estimates report for a company
:param:
symbol: stock symbol string, e.g. 'IBM'
:return:
a string of financial statements
'''
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
if isinstance(symbol, Contract):
contract = symbol
elif isinstance(symbol, str):
contract = new_stock_contract(symbol)
else:
raise TypeError(
"contract must be a contract object or string (for U.S. stocks only)."
)
__id = self.__get_new_request_id()
request = RequestDetails('reqFundamentalData',
'RESC-Analyst Estimates', contract)
response = ResponseDetails()
self.ipc_msg_dict[__id] = (request, response)
# IB Dividends ("456")
#
# This tick type provides four different comma-separated elements:
# The sum of dividends for the past 12 months (0.83 in the example below).
# The sum of dividends for the next 12 months (0.92 from the example below).
# The next dividend date (20130219 in the example below).
# The next single dividend amount (0.23 from the example below).
# Example: 0.83,0.92,20130219,0.23
self.connection.reqMktData(__id, contract, "456", False)
result = None
response.event.wait(max_wait_time)
if response.event.is_set():
if response.status == ResponseDetails.STATUS_FINISHED:
# TODO: convert the format
result = set(response.tick_str.split(','))
self.ipc_msg_dict.pop(__id)
return result
def get_contract_details(self, contract, max_wait_time=5):
""" Get contract details for a specified contract
Args:
contract: a legal IBPY Contract object or a string for U.S. stock only
Returns:
status: a reference to the tick data dictionary which will be updated with latest quote.
contract_details: a contractDetails instance
"""
if isinstance(contract, Contract):
pass
elif isinstance(contract, str):
contract = new_stock_contract(contract)
else:
raise TypeError(
"contract must be a contract object or string (for U.S. stocks only)."
)
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
__id = self.__get_new_request_id()
request = RequestDetails('reqContractDetails', '', contract)
response = ResponseDetails()
self.ipc_msg_dict[__id] = (request, response)
# False - indicating request live quotes instead of a snapshot
self.connection.reqContractDetails(__id, contract)
response.event.wait(max_wait_time)
contract_details = None
if response.event.is_set():
if response.status == ResponseDetails.STATUS_FINISHED:
if len(response.contract_list) > 0:
contract_details = copy(response.contract_list[0])
else:
pass
else:
pass
status = response.status
self.ipc_msg_dict.pop(__id)
return status, contract_details
def get_full_contract(self, contract):
""" Subscribe tick data for a specified contract
Args:
contract: a legal IBPY Contract object or a string for U.S. stock only
Returns:
tickerId: the ID of this request. this ID could be used to cancel request later.
tick_data: a reference to the tick data dictionary which will be updated with latest quote.
"""
status, contract_details = self.get_contract_details(contract)
new_contract = copy(contract_details.m_summary)
return status, new_contract
def request_market_depth(self, contract, num_rows=10):
"""
:param contract:
:param num_rows:
:return:
"""
if isinstance(contract, Contract):
pass
elif isinstance(contract, str):
contract = new_stock_contract(contract)
else:
raise TypeError(
"contract must be a contract object or string (for U.S. stocks only)."
)
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
__id = self.__get_new_request_id()
self.market_depth_buffer[__id] = MarketDepth(__id)
self.connection.reqMktDepth(__id, contract, num_rows)
return __id, self.market_depth_buffer[__id]
def cancel_market_depth(self, request_id):
"""
:param contract:
:param num_rows:
:return:
"""
if not self.connected:
raise RuntimeError('IB client is not connected to TWS')
if request_id in self.market_depth_buffer.keys():
self.connection.cancelMktDepth(request_id)
data = self.market_depth_buffer.pop(request_id)
return data
else:
raise ValueError("request_id is not found: %s" % request_id)
def do(event: Event, interval: int):
# 每interval检测flag是否为True
while not event.wait(interval):
logging.info("do sth.")
class RcpApi:
detect_win_callback = None
detect_fail_callback = None
detect_activity_callback = None
comms = None
msgListeners = {}
cmdSequenceQueue = Queue.Queue()
_command_queue = Queue.Queue()
sendCommandLock = RLock()
on_progress = lambda self, value: value
on_tx = lambda self, value: None
on_rx = lambda self, value: None
level_2_retries = DEFAULT_LEVEL2_RETRIES
msg_rx_timeout = DEFAULT_MSG_RX_TIMEOUT
_cmd_sequence_thread = None
_msg_rx_thread = None
_auto_detect_event = Event()
_auto_detect_busy = Event()
COMMAND_SEQUENCE_TIMEOUT = 1.0
COMMAND_DELIMETER = "\r\n"
def __init__(self,
settings,
on_disconnect=None,
on_connect=None,
**kwargs):
self.comms = kwargs.get('comms', self.comms)
self._running = Event()
self._running.clear()
self._enable_autodetect = Event()
self._enable_autodetect.set()
self._settings = settings
self._disconnect_listeners = []
self._connect_listeners = []
self.connected_version = None
if on_disconnect:
self.add_disconnect_listener(on_disconnect)
if on_connect:
self.add_connect_listener(on_connect)
@property
def connected(self):
'''
Returns True if we are currently connected to a device
:returns True if connected
'''
return self.connected_version is not None
def add_disconnect_listener(self, func):
self._disconnect_listeners.append(func)
def remove_disconnect_listener(self, func):
self._disconnect_listeners.remove(func)
def add_connect_listener(self, func):
self._connect_listeners.append(func)
def remove_connect_listener(self, func):
self._connect_listeners.remove(func)
def enable_autorecover(self):
Logger.debug("RCPAPI: Enabling auto recover")
self._enable_autodetect.set()
def disable_autorecover(self):
Logger.debug("RCPAPI: Disabling auto recover")
self._enable_autodetect.clear()
def recover_connection(self):
self.connected_version = None
self._notify_disconnect_listeners()
if self._enable_autodetect.is_set():
Logger.debug("RCPAPI: attempting to recover connection")
self.run_auto_detect()
def _notify_disconnect_listeners(self):
for listener in self._disconnect_listeners:
listener()
def _notify_connect_listeners(self):
for listener in self._connect_listeners:
listener()
def _start_message_rx_worker(self):
self._running.set()
t = Thread(target=self.msg_rx_worker)
t.daemon = True
t.start()
self._msg_rx_thread = t
def _shutdown_workers(self):
Logger.debug('RCPAPI: Stopping msg rx worker')
self._running.clear()
# this allows the auto detect worker to fall through if needed
self._auto_detect_event.set()
self._enable_autodetect.set()
self._auto_detect_worker.join()
self._msg_rx_thread.join()
self._cmd_sequence_thread.join()
def _start_cmd_sequence_worker(self):
t = Thread(target=self.cmd_sequence_worker)
t.daemon = True
t.start()
self._cmd_sequence_thread = t
def init_api(self, comms):
self.comms = comms
self._start_message_rx_worker()
self._start_cmd_sequence_worker()
self.start_auto_detect_worker()
Clock.schedule_interval(lambda dt: comms.keep_alive(),
COMMS_KEEP_ALIVE_TIMEOUT)
def shutdown_api(self):
self._shutdown_workers()
self.shutdown_comms()
def shutdown_comms(self):
Logger.debug('RCPAPI: shutting down comms')
try:
self.comms.close()
self.comms.device = None
except Exception as e:
Logger.warn('RCPAPI: Shutdown rx worker exception: {}'.format(e))
Logger.info(traceback.format_exc())
def detect_win(self, version_info):
self.level_2_retries = DEFAULT_LEVEL2_RETRIES
self.msg_rx_timeout = DEFAULT_MSG_RX_TIMEOUT
if self.detect_win_callback: self.detect_win_callback(version_info)
self.connected_version = version_info
self._notify_connect_listeners()
def run_auto_detect(self):
self.level_2_retries = AUTODETECT_LEVEL2_RETRIES
self.msg_rx_timeout = self.comms.CONNECT_TIMEOUT
self._auto_detect_event.set()
def addListener(self, messageName, callback):
listeners = self.msgListeners.get(messageName, None)
if listeners:
listeners.add(callback)
else:
listeners = set()
listeners.add(callback)
self.msgListeners[messageName] = listeners
def removeListener(self, messageName, callback):
listeners = self.msgListeners.get(messageName, None)
if listeners:
listeners.discard(callback)
def msg_rx_worker(self):
Logger.info('RCPAPI: msg_rx_worker starting')
comms = self.comms
error_count = 0
while self._running.is_set():
msg = None
try:
msg = comms.read_message()
if msg:
# clean incoming string, and drop illegal characters
msg = unicode(msg, errors='ignore')
msgJson = json.loads(msg, strict=False)
if 's' in msgJson:
Logger.trace('RCPAPI: Rx: ' + str(msg))
else:
Logger.debug('RCPAPI: Rx: ' + str(msg))
Clock.schedule_once(lambda dt: self.on_rx(True))
error_count = 0
for messageName in msgJson.keys():
Logger.trace('RCPAPI: processing message ' +
messageName)
listeners = self.msgListeners.get(messageName, None)
if listeners:
for listener in listeners:
try:
listener(msgJson)
except Exception as e:
Logger.error(
'RCPAPI: Message Listener Exception for'
)
Logger.debug(traceback.format_exc())
break
msg = ''
else:
sleep(NO_DATA_AVAILABLE_DELAY)
except PortNotOpenException:
Logger.debug("RCPAPI: Port not open...")
msg = ''
sleep(1.0)
except Exception as e:
Logger.warn(
'RCPAPI: Message rx worker exception: {} | {}'.format(
repr(msg), str(e)))
Logger.debug(traceback.format_exc())
msg = ''
error_count += 1
if error_count > 5 and not self._auto_detect_event.is_set():
Logger.warn(
"RCPAPI: Too many Rx exceptions; re-opening connection"
)
self.recover_connection()
self.connected_version = None
sleep(5)
else:
sleep(0.25)
safe_thread_exit()
Logger.info("RCPAPI: msg_rx_worker exiting")
def rcpCmdComplete(self, msgReply):
self.cmdSequenceQueue.put(msgReply)
def recoverTimeout(self):
Logger.warn('RCPAPI: POKE')
self.comms.write_message(' ')
def notifyProgress(self, count, total):
if self.on_progress:
Clock.schedule_once(lambda dt: self.on_progress(
(float(count) / float(total)) * 100))
def executeSingle(self, cmd, win_callback, fail_callback):
command = CommandSequence()
command.command_list = [cmd]
command.rootName = None
command.winCallback = win_callback
command.failCallback = fail_callback
self._command_queue.put(command)
def _queue_multiple(self, command_list, root_name, win_callback,
fail_callback):
command = CommandSequence()
command.command_list = command_list
command.rootName = root_name
command.winCallback = win_callback
command.failCallback = fail_callback
self._command_queue.put(command)
def cmd_sequence_worker(self):
Logger.info('RCPAPI: cmd_sequence_worker starting')
while self._running.is_set():
try:
# Block for 1 second for messages
command = self._command_queue.get(
True, RcpApi.COMMAND_SEQUENCE_TIMEOUT)
command_list = command.command_list
rootName = command.rootName
winCallback = command.winCallback
failCallback = command.failCallback
comms = self.comms
Logger.debug('RCPAPI: Execute Sequence begin')
if not comms.isOpen(): self.run_auto_detect()
q = self.cmdSequenceQueue
responseResults = {}
cmdCount = 0
cmdLength = len(command_list)
self.notifyProgress(cmdCount, cmdLength)
try:
for rcpCmd in command_list:
payload = rcpCmd.payload
index = rcpCmd.index
option = rcpCmd.option
last = rcpCmd.last
level2Retry = 0
name = rcpCmd.name
result = None
self.addListener(name, self.rcpCmdComplete)
while not result and level2Retry <= self.level_2_retries:
args = []
if payload is not None:
args.append(payload)
if index is not None:
args.append(index)
if option is not None:
args.append(option)
if last is not None:
args.append(last)
rcpCmd.cmd(*args)
retry = 0
while not result and retry < DEFAULT_READ_RETRIES:
try:
result = q.get(True, self.msg_rx_timeout)
msgName = result.keys()[0]
if not msgName == name:
Logger.warn(
'RCPAPI: rx message did not match expected name '
+ str(name) + '; ' + str(msgName))
result = None
except Exception as e:
Logger.warn(
'RCPAPI: Read message timeout waiting for {}'
.format(name))
self.recoverTimeout()
retry += 1
if not result:
Logger.warn('RCPAPI: Level 2 retry for (' +
str(level2Retry) + ') ' + name)
level2Retry += 1
if not result:
raise Exception('Timeout waiting for ' + name)
responseResults[name] = result[name]
self.removeListener(name, self.rcpCmdComplete)
cmdCount += 1
self.notifyProgress(cmdCount, cmdLength)
if rootName:
callback = self.callback_factory(
winCallback, {rootName: responseResults})
else:
callback = self.callback_factory(
winCallback, responseResults)
Clock.schedule_once(callback)
except CommsErrorException:
self.recover_connection()
self.connected_version = None
except Exception as detail:
Logger.error('RCPAPI: Command sequence exception: ' +
str(detail))
Logger.error(traceback.format_exc())
callback = self.callback_factory(failCallback, detail)
Clock.schedule_once(callback)
self.connected_version = None
self.recover_connection()
Logger.debug('RCPAPI: Execute Sequence complete')
except Queue.Empty:
pass
except Exception as e:
Logger.error('RCPAPI: Execute command exception ' + str(e))
Logger.info('RCPAPI: cmd_sequence_worker exiting')
safe_thread_exit()
def callback_factory(self, callback, *args):
"""
This function returns a function that when called, will call the argument callback with the remaining arguments
passed to this function. Weird, huh? We use it to handle the problem of lambda scoping in cmd_sequence_worker.
Basically, in cmd_sequence_worker we need to schedule the callbacks to happen in the UI thread, but
cmd_sequence_worker is running in a separate thread. So we use Clock.schedule_once to have it fire in the UI
thread. But if we're running a bunch of commands in a row, the variables that the callback function has scope to
will change out from underneath it. So we need to wrap our callback data in another function so we keep the
same scope .
:param callback:
:param args:
:return: Function
"""
return lambda dt: callback(*args)
def sendCommand(self, cmd):
try:
self.sendCommandLock.acquire()
rsp = None
comms = self.comms
cmdStr = json.dumps(cmd, separators=(',', ':')) + \
self.COMMAND_DELIMETER
Logger.debug('RCPAPI: Tx: ' + cmdStr)
comms.write_message(cmdStr)
except Exception as e:
Logger.error('RCPAPI: sendCommand exception ' + str(e))
Logger.error(traceback.format_exc())
self.recover_connection()
finally:
self.sendCommandLock.release()
Clock.schedule_once(lambda dt: self.on_tx(True))
def sendGet(self, name, index=None):
if index == None:
index = None
else:
index = str(index)
cmd = {name: index}
self.sendCommand(cmd)
def sendSet(self, name, payload, index=None):
if not index == None:
self.sendCommand({name: {str(index): payload}})
else:
self.sendCommand({name: payload})
def getRcpCfgCallback(self, cfg, rcpCfgJson, winCallback):
cfg.fromJson(rcpCfgJson)
winCallback(cfg)
def getRcpCfg(self, cfg, winCallback, failCallback):
def query_available_configs(capabilities_dict):
capabilities_dict = capabilities_dict.get('capabilities')
capabilities = Capabilities()
capabilities.from_json_dict(capabilities_dict,
self.connected_version)
cmdSequence = [
RcpCmd('ver', self.sendGetVersion),
RcpCmd('capabilities', self.getCapabilities),
RcpCmd('imuCfg', self.getImuCfg),
RcpCmd('gpsCfg', self.getGpsCfg),
RcpCmd('lapCfg', self.getLapCfg),
RcpCmd('trackCfg', self.getTrackCfg),
RcpCmd('canCfg', self.getCanCfg),
RcpCmd('obd2Cfg', self.getObd2Cfg),
RcpCmd('connCfg', self.getConnectivityCfg),
RcpCmd('trackDb', self.getTrackDb)
]
if capabilities.has_can_channel:
cmdSequence.append(
RcpCmd('canChanCfg', self.get_can_channels_config))
if capabilities.has_script:
cmdSequence.append(RcpCmd('scriptCfg', self.getScript))
if capabilities.has_analog:
cmdSequence.append(RcpCmd('analogCfg', self.getAnalogCfg))
if capabilities.has_timer:
cmdSequence.append(RcpCmd('timerCfg', self.getTimerCfg))
if capabilities.has_gpio:
cmdSequence.append(RcpCmd('gpioCfg', self.getGpioCfg))
if capabilities.has_pwm:
cmdSequence.append(RcpCmd('pwmCfg', self.getPwmCfg))
if capabilities.has_wifi:
cmdSequence.append(RcpCmd('wifiCfg', self.get_wifi_config))
self._queue_multiple(
cmdSequence, 'rcpCfg', lambda rcpJson: self.getRcpCfgCallback(
cfg, rcpJson, winCallback), failCallback)
# First we need to get capabilities, then figure out what to query
self.executeSingle(RcpCmd('capabilities', self.getCapabilities),
query_available_configs, failCallback)
def get_capabilities(self, success_cb, fail_cb):
# Capabilities object also needs version info
self.executeSingle(RcpCmd('capabilities', self.getCapabilities),
success_cb, fail_cb)
def writeRcpCfg(self, cfg, winCallback=None, failCallback=None):
cmdSequence = []
connCfg = cfg.connectivityConfig
if connCfg.stale:
cmdSequence.append(
RcpCmd('setConnCfg', self.setConnectivityCfg,
connCfg.toJson()))
gpsCfg = cfg.gpsConfig
if gpsCfg.stale:
cmdSequence.append(
RcpCmd('setGpsCfg', self.setGpsCfg, gpsCfg.toJson()))
lapCfg = cfg.lapConfig
if lapCfg.stale:
cmdSequence.append(
RcpCmd('setLapCfg', self.setLapCfg, lapCfg.toJson()))
imuCfg = cfg.imuConfig
for i in range(imuCfg.channelCount):
imuChannel = imuCfg.channels[i]
if imuChannel.stale:
cmdSequence.append(
RcpCmd('setImuCfg', self.setImuCfg, imuChannel.toJson(),
i))
analogCfg = cfg.analogConfig
for i in range(analogCfg.channelCount):
analogChannel = analogCfg.channels[i]
if analogChannel.stale:
cmdSequence.append(
RcpCmd('setAnalogCfg', self.setAnalogCfg,
analogChannel.toJson(), i))
timerCfg = cfg.timerConfig
for i in range(timerCfg.channelCount):
timerChannel = timerCfg.channels[i]
if timerChannel.stale:
cmdSequence.append(
RcpCmd('setTimerCfg', self.setTimerCfg,
timerChannel.toJson(), i))
gpioCfg = cfg.gpioConfig
for i in range(gpioCfg.channelCount):
gpioChannel = gpioCfg.channels[i]
if gpioChannel.stale:
cmdSequence.append(
RcpCmd('setGpioCfg', self.setGpioCfg, gpioChannel.toJson(),
i))
pwmCfg = cfg.pwmConfig
for i in range(pwmCfg.channelCount):
pwmChannel = pwmCfg.channels[i]
if pwmChannel.stale:
cmdSequence.append(
RcpCmd('setPwmCfg', self.setPwmCfg, pwmChannel.toJson(),
i))
canCfg = cfg.canConfig
if canCfg.stale:
cmdSequence.append(
RcpCmd('setCanCfg', self.setCanCfg, canCfg.toJson()))
obd2Cfg = cfg.obd2Config
if obd2Cfg.stale:
self.sequence_write_obd2_channels(obd2Cfg.toJson(), cmdSequence)
can_channels = cfg.can_channels
if can_channels.stale:
self.sequence_write_can_channels(can_channels.to_json_dict(),
cmdSequence)
trackCfg = cfg.trackConfig
if trackCfg.stale:
cmdSequence.append(
RcpCmd('setTrackCfg', self.setTrackCfg, trackCfg.toJson()))
scriptCfg = cfg.scriptConfig
if scriptCfg.stale:
self.sequenceWriteScript(scriptCfg.toJson(), cmdSequence)
trackDb = cfg.trackDb
if trackDb.stale:
self.sequenceWriteTrackDb(trackDb.toJson(), cmdSequence)
wifi_config = cfg.wifi_config
if wifi_config.stale:
cmdSequence.append(
RcpCmd('setWifiCfg', self.set_wifi_config,
wifi_config.to_json()))
cmdSequence.append(RcpCmd('flashCfg', self.sendFlashConfig))
self._queue_multiple(cmdSequence, 'setRcpCfg', winCallback,
failCallback)
def resetDevice(self, bootloader=False, reset_delay=0):
if bootloader:
loaderint = 1
else:
loaderint = 0
self.sendCommand(
{'sysReset': {
'loader': loaderint,
'delay': reset_delay
}})
def getAnalogCfg(self, channelId=None):
self.sendGet('getAnalogCfg', channelId)
def setAnalogCfg(self, analogCfg, channelId):
self.sendSet('setAnalogCfg', analogCfg, channelId)
def getImuCfg(self, channelId=None, success_cb=None, fail_cb=None):
if success_cb:
self.executeSingle(RcpCmd('imuCfg', self.getImuCfg), success_cb,
fail_cb)
else:
self.sendGet('getImuCfg', channelId)
def setImuCfg(self, imuCfg, channelId):
self.sendSet('setImuCfg', imuCfg, channelId)
def getLapCfg(self):
self.sendGet('getLapCfg', None)
def setLapCfg(self, lapCfg):
self.sendSet('setLapCfg', lapCfg)
def getGpsCfg(self):
self.sendGet('getGpsCfg', None)
def setGpsCfg(self, gpsCfg):
self.sendSet('setGpsCfg', gpsCfg)
def getTimerCfg(self, channelId=None):
self.sendGet('getTimerCfg', channelId)
def setTimerCfg(self, timerCfg, channelId):
self.sendSet('setTimerCfg', timerCfg, channelId)
def setGpioCfg(self, gpioCfg, channelId):
self.sendSet('setGpioCfg', gpioCfg, channelId)
def getGpioCfg(self, channelId=None):
self.sendGet('getGpioCfg', channelId)
def getPwmCfg(self, channelId=None):
self.sendGet('getPwmCfg', channelId)
def setPwmCfg(self, pwmCfg, channelId):
self.sendSet('setPwmCfg', pwmCfg, channelId)
def getTrackCfg(self, success_cb=None, fail_cb=None):
if success_cb is None:
self.sendGet('getTrackCfg', None)
else:
self.executeSingle(RcpCmd('trackCfg', self.getTrackCfg),
success_cb, fail_cb)
def setTrackCfg(self, trackCfg):
self.sendSet('setTrackCfg', trackCfg)
def getCanCfg(self):
self.sendGet('getCanCfg', None)
def setCanCfg(self, canCfg):
self.sendSet('setCanCfg', canCfg)
def getObd2Cfg(self):
self.sendGet('getObd2Cfg', None)
def sequence_write_obd2_channels(self, obd2_channels_json_dict,
cmd_sequence):
"""
queue writing of all OBD2 channels
"""
channels = obd2_channels_json_dict['obd2Cfg']['pids']
enabled = obd2_channels_json_dict['obd2Cfg']['en']
channels_len = len(channels)
if channels is not None:
index = 0
if channels_len > 0:
for c in channels:
cmd_sequence.append(
RcpCmd('setObd2Cfg', self.set_obd2_channel_config, [c],
index, enabled, index == channels_len - 1))
index += 1
else:
# if we've removed all channels, send message with empty channel array
cmd_sequence.append(
RcpCmd('setObd2Cfg', self.set_obd2_channel_config, [],
index, enabled, True))
def set_obd2_channel_config(self, obd2_channels, index, enabled, last):
"""
Write a single OBD2 channel configuration by index
"""
payload = {'en': enabled, 'index': index, 'pids': obd2_channels}
if last == True:
payload['last'] = True
msg = {'setObd2Cfg': payload}
return self.sendCommand(msg)
def sequence_write_can_channels(self, can_channels_json_dict,
cmd_sequence):
"""
queue writing of all can channels
"""
channels = can_channels_json_dict['canChanCfg']['chans']
enabled = can_channels_json_dict['canChanCfg']['en']
channels_len = len(channels)
if channels is not None:
index = 0
if channels_len > 0:
for c in channels:
cmd_sequence.append(
RcpCmd('setCanChanCfg', self.set_can_channel_config,
[c], index, enabled, index == channels_len - 1))
index += 1
else:
# if we've removed all channels, send message with empty channel array
cmd_sequence.append(
RcpCmd('setCanChanCfg', self.set_can_channel_config, [],
index, enabled, True))
def set_can_channel_config(self, can_channels, index, enabled, last):
"""
Write a single CAN channel configuration by index
"""
payload = {'en': enabled, 'index': index, 'chans': can_channels}
if last == True:
payload['last'] = True
msg = {'setCanChanCfg': payload}
return self.sendCommand(msg)
def get_can_channels_config(self):
self.sendGet('getCanChanCfg', None)
def getConnectivityCfg(self):
self.sendGet('getConnCfg', None)
def setConnectivityCfg(self, connCfg):
self.sendSet('setConnCfg', connCfg)
def get_wifi_config(self):
self.sendGet('getWifiCfg', None)
def set_wifi_config(self, wifi_config):
self.sendSet('setWifiCfg', wifi_config)
def start_telemetry(self, rate):
self.sendSet('setTelemetry', {'rate': rate})
def stop_telemetry(self):
self.sendSet('setTelemetry', {'rate': 0})
def getScript(self):
self.sendGet('getScriptCfg', None)
def setScriptPage(self, scriptPage, page, mode):
self.sendCommand(
{'setScriptCfg': {
'data': scriptPage,
'page': page,
'mode': mode
}})
def get_status(self, success_cb=None, fail_cb=None):
if success_cb is not None:
self.executeSingle(RcpCmd('status', self.getStatus), success_cb,
fail_cb)
else:
self.sendGet('getStatus', None)
def sequenceWriteScript(self, scriptCfg, cmdSequence):
page = 0
script = scriptCfg['scriptCfg']['data']
while True:
if len(script) >= 256:
scr = script[:256]
script = script[256:]
mode = SCRIPT_ADD_MODE_IN_PROGRESS if len(
script) > 0 else SCRIPT_ADD_MODE_COMPLETE
cmdSequence.append(
RcpCmd('setScriptCfg', self.setScriptPage, scr, page,
mode))
page = page + 1
else:
cmdSequence.append(
RcpCmd('setScriptCfg', self.setScriptPage, script, page,
SCRIPT_ADD_MODE_COMPLETE))
break
def sendRunScript(self):
self.sendCommand({'runScript': None})
def runScript(self, winCallback, failCallback):
self.executeSingle(RcpCmd('runScript', self.sendRunScript),
winCallback, failCallback)
def setLogfileLevel(self, level, winCallback=None, failCallback=None):
def setLogfileLevelCmd():
self.sendCommand({'setLogfileLevel': {'level': level}})
if winCallback and failCallback:
self.executeSingle(RcpCmd('logfileLevel', setLogfileLevelCmd),
winCallback, failCallback)
else:
setLogfileLevelCmd()
def getLogfile(self, winCallback=None, failCallback=None):
def getLogfileCmd():
self.sendCommand({'getLogfile': None})
if winCallback and failCallback:
self.executeSingle(RcpCmd('logfile', getLogfileCmd), winCallback,
failCallback)
else:
getLogfileCmd()
def sendFlashConfig(self):
self.sendCommand({'flashCfg': None})
def sequenceWriteTrackDb(self, tracksDbJson, cmdSequence):
trackDbJson = tracksDbJson.get('trackDb')
if trackDbJson:
index = 0
tracksJson = trackDbJson.get('tracks')
if tracksJson:
trackCount = len(tracksJson)
for trackJson in tracksJson:
mode = TRACK_ADD_MODE_IN_PROGRESS if index < trackCount - 1 else TRACK_ADD_MODE_COMPLETE
cmdSequence.append(
RcpCmd('addTrackDb', self.addTrackDb, trackJson, index,
mode))
index += 1
else:
cmdSequence.append(
RcpCmd('addTrackDb', self.addTrackDb, [], index,
TRACK_ADD_MODE_COMPLETE))
def addTrackDb(self, trackJson, index, mode):
return self.sendCommand(
{'addTrackDb': {
'index': index,
'mode': mode,
'track': trackJson
}})
def getTrackDb(self):
self.sendGet('getTrackDb')
def sendGetVersion(self):
self.sendCommand({"getVer": None})
def getVersion(self, winCallback, failCallback):
self.executeSingle(RcpCmd('ver', self.sendGetVersion), winCallback,
failCallback)
def getCapabilities(self):
self.sendGet('getCapabilities')
def getStatus(self):
self.sendGet('getStatus')
def sendCalibrateImu(self):
self.sendCommand({"calImu": 1})
def calibrate_imu(self, winCallback, failCallback):
cmd_sequence = []
cmd_sequence.append(RcpCmd('calImu', self.sendCalibrateImu))
cmd_sequence.append(RcpCmd('flashCfg', self.sendFlashConfig))
self._queue_multiple(cmd_sequence, 'calImu', winCallback, failCallback)
def get_meta(self):
Logger.debug("RCPAPI: sending meta")
self.sendCommand({'getMeta': None})
def set_active_track(self, track):
Logger.debug("RCPAPI: setting active track: {}".format(track))
track_json = track.toJson()
self.sendCommand({'setActiveTrack': {'track': track_json}})
def sample(self, include_meta=False):
if include_meta:
self.sendCommand({'s': {'meta': 1}})
else:
self.sendCommand({'s': 0})
def is_firmware_update_supported(self):
"""
Returns True if this connection supports firmware upgrading
"""
return self.comms and not self.comms.is_wireless()
@property
def is_wireless_connection(self):
"""
Returns True if connection is wireless, or false if wired, such as USB
"""
return self.comms and self.comms.is_wireless()
def start_auto_detect_worker(self):
self._auto_detect_event.clear()
t = Thread(target=self.auto_detect_worker)
t.daemon = True
t.start()
self._auto_detect_worker = t
def auto_detect_worker(self):
Logger.info('RCPAPI: auto_detect_worker starting')
class VersionResult(object):
version_json = None
def on_ver_win(value):
version_result.version_json = value
version_result_event.set()
while self._running.is_set():
self._auto_detect_event.wait()
self._auto_detect_event.clear()
self._enable_autodetect.wait()
# check again if we're shutting down
# to prevent a needless re-detection attempt
if not self._running.is_set():
break
try:
Logger.debug("RCPAPI: Starting auto-detect")
self._auto_detect_busy.set()
self.sendCommandLock.acquire()
self.addListener("ver", on_ver_win)
comms = self.comms
if comms and comms.isOpen():
comms.close()
version_result = VersionResult()
version_result_event = Event()
version_result_event.clear()
if comms.device:
devices = [comms.device]
else:
devices = comms.get_available_devices()
last_known_device = self._settings.userPrefs.get_pref(
'preferences', 'last_known_device')
# if there was a last known device try this one first.
if last_known_device:
Logger.info(
'RCPAPI: trying last known device first: {}'.
format(last_known_device))
# ensure we remove it from the existing list
try:
devices.remove(last_known_device)
except ValueError:
pass
devices = [last_known_device] + devices
Logger.debug('RCPAPI: Searching for device')
testVer = VersionConfig()
for device in devices:
try:
Logger.debug('RCPAPI: Trying ' + str(device))
if self.detect_activity_callback:
self.detect_activity_callback(str(device))
comms.device = device
comms.open()
self.sendGetVersion()
version_result_event.wait(2)
version_result_event.clear()
if version_result.version_json != None:
testVer.fromJson(
version_result.version_json.get('ver', None))
if testVer.is_valid:
break # we found something!
else:
try:
Logger.debug('RCPAPI: Giving up on ' +
str(device))
comms.close()
finally:
pass
except Exception as detail:
Logger.error('RCPAPI: Not found on ' + str(device) +
" " + str(detail))
Logger.error(traceback.format_exc())
try:
comms.close()
finally:
pass
if testVer.is_valid:
Logger.debug("RCPAPI: Found device version " +
str(testVer) + " on port: " +
str(comms.device))
self.detect_win(testVer)
self._auto_detect_event.clear()
self._settings.userPrefs.set_pref('preferences',
'last_known_device',
comms.device)
else:
Logger.debug('RCPAPI: Did not find device')
comms.close()
comms.device = None
if self.detect_fail_callback: self.detect_fail_callback()
except Exception as e:
Logger.error('RCPAPI: Error running auto detect: ' + str(e))
Logger.error(traceback.format_exc())
if self.detect_fail_callback: self.detect_fail_callback()
finally:
Logger.debug("RCPAPI: auto detect finished. port=" +
str(comms.device))
self._auto_detect_busy.clear()
self.removeListener("ver", on_ver_win)
self.sendCommandLock.release()
comms.device = None
sleep(AUTODETECT_COOLOFF_TIME)
safe_thread_exit()
Logger.debug('RCPAPI: auto_detect_worker exiting')
class ChipStack(object):
def __init__(self, installDefaultLogHandler=True):
self.networkLock = Lock()
self.completeEvent = Event()
self._ChipStackLib = None
self._chipDLLPath = None
self.devMgr = None
self.callbackRes = None
self._activeLogFunct = None
self.addModulePrefixToLogMessage = True
# Locate and load the chip shared library.
self._loadLib()
# Arrange to log output from the chip library to a python logger object with the
# name 'chip.ChipStack'. If desired, applications can override this behavior by
# setting self.logger to a different python logger object, or by calling setLogFunct()
# with their own logging function.
self.logger = logging.getLogger(__name__)
self.setLogFunct(self.defaultLogFunct)
# Determine if there are already handlers installed for the logger. Python 3.5+
# has a method for this; on older versions the check has to be done manually.
if hasattr(self.logger, "hasHandlers"):
hasHandlers = self.logger.hasHandlers()
else:
hasHandlers = False
logger = self.logger
while logger is not None:
if len(logger.handlers) > 0:
hasHandlers = True
break
if not logger.propagate:
break
logger = logger.parent
# If a logging handler has not already been initialized for 'chip.ChipStack',
# or any one of its parent loggers, automatically configure a handler to log to
# stdout. This maintains compatibility with a number of applications which expect
# chip log output to go to stdout by default.
#
# This behavior can be overridden in a variety of ways:
# - Initialize a different log handler before ChipStack is initialized.
# - Pass installDefaultLogHandler=False when initializing ChipStack.
# - Replace the StreamHandler on self.logger with a different handler object.
# - Set a different Formatter object on the existing StreamHandler object.
# - Reconfigure the existing ChipLogFormatter object.
# - Configure chip to call an application-specific logging function by
# calling self.setLogFunct().
# - Call self.setLogFunct(None), which will configure the chip library
# to log directly to stdout, bypassing python altogether.
#
if installDefaultLogHandler and not hasHandlers:
logHandler = logging.StreamHandler(stream=sys.stdout)
logHandler.setFormatter(ChipLogFormatter())
self.logger.addHandler(logHandler)
self.logger.setLevel(logging.DEBUG)
def HandleComplete(appState, reqState):
self.callbackRes = True
self.completeEvent.set()
def HandleError(appState, reqState, err, devStatusPtr):
self.callbackRes = self.ErrorToException(err, devStatusPtr)
self.completeEvent.set()
self.cbHandleComplete = _CompleteFunct(HandleComplete)
self.cbHandleError = _ErrorFunct(HandleError)
self.blockingCB = None # set by other modules(BLE) that require service by thread while thread blocks.
# Initialize the chip library
res = self._ChipStackLib.nl_Chip_Stack_Init()
if res != 0:
raise self._ChipStack.ErrorToException(res)
@property
def defaultLogFunct(self):
"""Returns a python callable which, when called, logs a message to the python logger object
currently associated with the ChipStack object.
The returned function is suitable for passing to the setLogFunct() method."""
def logFunct(timestamp, timestampUSec, moduleName, logCat, message):
moduleName = ChipUtility.CStringToString(moduleName)
message = ChipUtility.CStringToString(message)
if self.addModulePrefixToLogMessage:
message = "WEAVE:%s: %s" % (moduleName, message)
logLevel = LogCategory.categoryToLogLevel(logCat)
msgAttrs = {
"chip-module": moduleName,
"timestamp": timestamp,
"timestamp-usec": timestampUSec,
}
self.logger.log(logLevel, message, extra=msgAttrs)
return logFunct
def setLogFunct(self, logFunct):
"""Set the function used by the chip library to log messages.
The supplied object must be a python callable that accepts the following
arguments:
timestamp (integer)
timestampUS (integer)
module name (encoded UTF-8 string)
log category (integer)
message (encoded UTF-8 string)
Specifying None configures the chip library to log directly to stdout."""
if logFunct is None:
logFunct = 0
if not isinstance(logFunct, _LogMessageFunct):
logFunct = _LogMessageFunct(logFunct)
with self.networkLock:
# NOTE: ChipStack must hold a reference to the CFUNCTYPE object while it is
# set. Otherwise it may get garbage collected, and logging calls from the
# chip library will fail.
self._activeLogFunct = logFunct
self._ChipStackLib.nl_Chip_Stack_SetLogFunct(logFunct)
def Shutdown(self):
self._ChipStack.Call(lambda: self._dmLib.nl_Chip_Stack_Shutdown())
self.networkLock = None
self.completeEvent = None
self._ChipStackLib = None
self._chipDLLPath = None
self.devMgr = None
self.callbackRes = None
def Call(self, callFunct):
# throw error if op in progress
self.callbackRes = None
self.completeEvent.clear()
with self.networkLock:
res = callFunct()
self.completeEvent.set()
if res == 0 and self.callbackRes != None:
return self.callbackRes
return res
def CallAsync(self, callFunct):
# throw error if op in progress
self.callbackRes = None
self.completeEvent.clear()
with self.networkLock:
res = callFunct()
if res != 0:
self.completeEvent.set()
raise self.ErrorToException(res)
while not self.completeEvent.isSet():
if self.blockingCB:
self.blockingCB()
self.completeEvent.wait(0.05)
if isinstance(self.callbackRes, ChipStackException):
raise self.callbackRes
return self.callbackRes
def ErrorToException(self, err, devStatusPtr=None):
if err == 4044 and devStatusPtr:
devStatus = devStatusPtr.contents
msg = ChipUtility.CStringToString(
(self._ChipStackLib.nl_Chip_Stack_StatusReportToString(
devStatus.ProfileId, devStatus.StatusCode)))
sysErrorCode = (devStatus.SysErrorCode if
(devStatus.SysErrorCode != 0) else None)
if sysErrorCode != None:
msg = msg + " (system err %d)" % (sysErrorCode)
return DeviceError(devStatus.ProfileId, devStatus.StatusCode,
sysErrorCode, msg)
else:
return ChipStackError(
err,
ChipUtility.CStringToString(
(self._ChipStackLib.nl_Chip_Stack_ErrorToString(err))),
)
def LocateChipDLL(self):
if self._chipDLLPath:
return self._chipDLLPath
scriptDir = os.path.dirname(os.path.abspath(__file__))
# When properly installed in the chip package, the Chip Device Manager DLL will
# be located in the package root directory, along side the package's
# modules.
dmDLLPath = os.path.join(scriptDir, ChipStackDLLBaseName)
if os.path.exists(dmDLLPath):
self._chipDLLPath = dmDLLPath
return self._chipDLLPath
# For the convenience of developers, search the list of parent paths relative to the
# running script looking for an CHIP build directory containing the Chip Device
# Manager DLL. This makes it possible to import and use the ChipDeviceMgr module
# directly from a built copy of the CHIP source tree.
buildMachineGlob = "%s-*-%s*" % (platform.machine(),
platform.system().lower())
relDMDLLPathGlob = os.path.join(
"build",
buildMachineGlob,
"src/controller/python/.libs",
ChipStackDLLBaseName,
)
for dir in self._AllDirsToRoot(scriptDir):
dmDLLPathGlob = os.path.join(dir, relDMDLLPathGlob)
for dmDLLPath in glob.glob(dmDLLPathGlob):
if os.path.exists(dmDLLPath):
self._chipDLLPath = dmDLLPath
return self._chipDLLPath
raise Exception(
"Unable to locate Chip Device Manager DLL (%s); expected location: %s"
% (ChipStackDLLBaseName, scriptDir))
# ----- Private Members -----
def _AllDirsToRoot(self, dir):
dir = os.path.abspath(dir)
while True:
yield dir
parent = os.path.dirname(dir)
if parent == "" or parent == dir:
break
dir = parent
def _loadLib(self):
if self._ChipStackLib is None:
self._ChipStackLib = CDLL(self.LocateChipDLL())
self._ChipStackLib.nl_Chip_Stack_Init.argtypes = []
self._ChipStackLib.nl_Chip_Stack_Init.restype = c_uint32
self._ChipStackLib.nl_Chip_Stack_Shutdown.argtypes = []
self._ChipStackLib.nl_Chip_Stack_Shutdown.restype = c_uint32
self._ChipStackLib.nl_Chip_Stack_StatusReportToString.argtypes = [
c_uint32,
c_uint16,
]
self._ChipStackLib.nl_Chip_Stack_StatusReportToString.restype = c_char_p
self._ChipStackLib.nl_Chip_Stack_ErrorToString.argtypes = [
c_uint32
]
self._ChipStackLib.nl_Chip_Stack_ErrorToString.restype = c_char_p
self._ChipStackLib.nl_Chip_Stack_SetLogFunct.argtypes = [
_LogMessageFunct
]
self._ChipStackLib.nl_Chip_Stack_SetLogFunct.restype = c_uint32
class Dot:
"""
The internal base class for the implementation of a "button" or "buttons".
"""
def __init__(self, color, square, border, visible):
self._color = color
self._square = square
self._border = border
self._visible = visible
self._is_pressed_event = Event()
self._is_released_event = Event()
self._is_moved_event = Event()
self._is_swiped_event = Event()
self._is_double_pressed_event = Event()
self._when_pressed = None
self._when_pressed_background = False
self._when_double_pressed = None
self._when_double_pressed_background = False
self._when_released = None
self._when_released_background = False
self._when_moved = None
self._when_moved_background = False
self._when_swiped = None
self._when_swiped_background = False
self._when_rotated = None
self._when_rotated_background = False
self._is_pressed = False
self._position = None
self._double_press_time = 0.3
self._rotation_segments = 8
@property
def is_pressed(self):
"""
Returns ``True`` if the button is pressed (or held).
"""
return self._is_pressed
@property
def value(self):
"""
Returns a 1 if ``.is_pressed``, 0 if not.
"""
return 1 if self.is_pressed else 0
@property
def values(self):
"""
Returns an infinite generator constantly yielding the current value.
"""
while True:
yield self.value
@property
def position(self):
"""
Returns an instance of :class:`BlueDotPosition` representing the
current or last position the button was pressed, held or
released.
.. note::
If the button is released (and inactive), :attr:`position` will
return the position where it was released, until it is pressed
again. If the button has never been pressed :attr:`position` will
return ``None``.
"""
return self._position
@property
def when_pressed(self):
"""
Sets or returns the function which is called when the button is pressed.
The function should accept 0 or 1 parameters, if the function accepts 1 parameter an
instance of :class:`BlueDotPosition` will be returned representing where the button was pressed.
The following example will print a message to the screen when the button is pressed::
from bluedot import BlueDot
def dot_was_pressed():
print("The button was pressed")
bd = BlueDot()
bd.when_pressed = dot_was_pressed
This example shows how the position of where the button was pressed can be obtained::
from bluedot import BlueDot
def dot_was_pressed(pos):
print("The button was pressed at pos x={} y={}".format(pos.x, pos.y))
bd = BlueDot()
bd.when_pressed = dot_was_pressed
The function will be run in the same thread and block, to run in a separate
thread use `set_when_pressed(function, background=True)`
"""
return self._when_pressed
@when_pressed.setter
def when_pressed(self, value):
self.set_when_pressed(value)
def set_when_pressed(self, callback, background=False):
"""
Sets the function which is called when the button is pressed.
:param Callable callback:
The function to call, setting to `None` will stop the callback.
:param bool background:
If set to `True` the function will be run in a separate thread
and it will return immediately. The default is `False`.
"""
self._when_pressed = callback
self._when_pressed_background = background
@property
def when_double_pressed(self):
"""
Sets or returns the function which is called when the button is double pressed.
The function should accept 0 or 1 parameters, if the function accepts 1 parameter an
instance of :class:`BlueDotPosition` will be returned representing where the button was
pressed the second time.
The function will be run in the same thread and block, to run in a separate
thread use `set_when_double_pressed(function, background=True)`
.. note::
The double press event is fired before the 2nd press event e.g. events would be
appear in the order, pressed, released, double pressed, pressed.
"""
return self._when_double_pressed
@when_double_pressed.setter
def when_double_pressed(self, value):
self.set_when_double_pressed(value)
def set_when_double_pressed(self, callback, background=False):
"""
Sets the function which is called when the button is double pressed.
:param Callable callback:
The function to call, setting to `None` will stop the callback.
:param bool background:
If set to `True` the function will be run in a separate thread
and it will return immediately. The default is `False`.
"""
self._when_double_pressed = callback
self._when_double_pressed_background = background
@property
def double_press_time(self):
"""
Sets or returns the time threshold in seconds for a double press. Defaults to 0.3.
"""
return self._double_press_time
@double_press_time.setter
def double_press_time(self, value):
self._double_press_time = value
@property
def when_released(self):
"""
Sets or returns the function which is called when the button is released.
The function should accept 0 or 1 parameters, if the function accepts 1 parameter an
instance of :class:`BlueDotPosition` will be returned representing where the button was held
when it was released.
The function will be run in the same thread and block, to run in a separate
thread use `set_when_released(function, background=True)`
"""
return self._when_released
@when_released.setter
def when_released(self, value):
self.set_when_released(value)
def set_when_released(self, callback, background=False):
"""
Sets the function which is called when the button is released.
:param Callable callback:
The function to call, setting to `None` will stop the callback.
:param bool background:
If set to `True` the function will be run in a separate thread
and it will return immediately. The default is `False`.
"""
self._when_released = callback
self._when_released_background = background
@property
def when_moved(self):
"""
Sets or returns the function which is called when the position the button is pressed is moved.
The function should accept 0 or 1 parameters, if the function accepts 1 parameter an
instance of :class:`BlueDotPosition` will be returned representing the new position of where the
Blue Dot is held.
The function will be run in the same thread and block, to run in a separate
thread use `set_when_moved(function, background=True)`
"""
return self._when_moved
@when_moved.setter
def when_moved(self, value):
self.set_when_moved(value)
def set_when_moved(self, callback, background=False):
"""
Sets the function which is called when the position the button is pressed is moved.
:param Callable callback:
The function to call, setting to `None` will stop the callback.
:param bool background:
If set to `True` the function will be run in a separate thread
and it will return immediately. The default is `False`.
"""
self._when_moved = callback
self._when_moved_background = background
@property
def when_swiped(self):
"""
Sets or returns the function which is called when the button is swiped.
The function should accept 0 or 1 parameters, if the function accepts 1 parameter an
instance of :class:`BlueDotSwipe` will be returned representing the how the button was
swiped.
The function will be run in the same thread and block, to run in a separate
thread use `set_when_swiped(function, background=True)`
"""
return self._when_swiped
@when_swiped.setter
def when_swiped(self, value):
self.set_when_swiped(value)
def set_when_swiped(self, callback, background=False):
"""
Sets the function which is called when the position the button is swiped.
:param Callable callback:
The function to call, setting to `None` will stop the callback.
:param bool background:
If set to `True` the function will be run in a separate thread
and it will return immediately. The default is `False`.
"""
self._when_swiped = callback
self._when_swiped_background = background
@property
def rotation_segments(self):
"""
Sets or returns the number of virtual segments the button is split into for rotating.
Defaults to 8.
"""
return self._rotation_segments
@rotation_segments.setter
def rotation_segments(self, value):
self._rotation_segments = value
@property
def when_rotated(self):
"""
Sets or returns the function which is called when the button is rotated (like an
iPod clock wheel).
The function should accept 0 or 1 parameters, if the function accepts 1 parameter an
instance of :class:`BlueDotRotation` will be returned representing how the button was
rotated.
The function will be run in the same thread and block, to run in a separate
thread use `set_when_rotated(function, background=True)`
"""
return self._when_rotated
@when_rotated.setter
def when_rotated(self, value):
self.set_when_rotated(value)
def set_when_rotated(self, callback, background=False):
"""
Sets the function which is called when the position the button is rotated (like an
iPod clock wheel).
:param Callable callback:
The function to call, setting to `None` will stop the callback.
:param bool background:
If set to `True` the function will be run in a separate thread
and it will return immediately. The default is `False`.
"""
self._when_rotated = callback
self._when_rotated_background = background
@property
def color(self):
"""
Sets or returns the color of the dot. Defaults to BLUE.
An instance of :class:`.colors.Color` is returned.
Value can be set as a :class:`.colors.Color` object, a hex color value
in the format `#rrggbb` or `#rrggbbaa`, a tuple of `(red, green, blue)`
or `(red, green, blue, alpha)` values between `0` & `255` or a text
description of the color, e.g. "red".
A dictionary of available colors can be obtained from `bluedot.COLORS`.
"""
return self._color
@color.setter
def color(self, value):
self._color = parse_color(value)
@property
def square(self):
"""
When set to `True` the 'dot' is made square. Default is `False`.
"""
return self._square
@square.setter
def square(self, value):
self._square = value
@property
def border(self):
"""
When set to `True` adds a border to the dot. Default is `False`.
"""
return self._border
@border.setter
def border(self, value):
self._border = value
@property
def visible(self):
"""
When set to `False` the dot will be hidden. Default is `True`.
.. note::
Events (press, release, moved) are still sent from the dot
when it is not visible.
"""
return self._visible
@visible.setter
def visible(self, value):
self._visible = value
def wait_for_press(self, timeout=None):
"""
Waits until a Blue Dot is pressed.
Returns ``True`` if the button was pressed.
:param float timeout:
Number of seconds to wait for a Blue Dot to be pressed, if ``None``
(the default), it will wait indefinetly.
"""
return self._is_pressed_event.wait(timeout)
def wait_for_double_press(self, timeout=None):
"""
Waits until a Blue Dot is double pressed.
Returns ``True`` if the button was double pressed.
:param float timeout:
Number of seconds to wait for a Blue Dot to be double pressed, if ``None``
(the default), it will wait indefinetly.
"""
return self._is_double_pressed_event.wait(timeout)
def wait_for_release(self, timeout=None):
"""
Waits until a Blue Dot is released.
Returns ``True`` if the button was released.
:param float timeout:
Number of seconds to wait for a Blue Dot to be released, if ``None``
(the default), it will wait indefinetly.
"""
return self._is_released_event.wait(timeout)
def wait_for_move(self, timeout=None):
"""
Waits until the position where the button is pressed is moved.
Returns ``True`` if the position pressed on the button was moved.
:param float timeout:
Number of seconds to wait for the position that the button
is pressed to move, if ``None`` (the default), it will wait indefinetly.
"""
return self._is_moved_event.wait(timeout)
def wait_for_swipe(self, timeout=None):
"""
Waits until the button is swiped.
Returns ``True`` if the button was swiped.
:param float timeout:
Number of seconds to wait for the button to be swiped, if ``None``
(the default), it will wait indefinetly.
"""
return self._is_swiped_event.wait(timeout)
def press(self, position):
"""
Processes any "pressed" events associated with this dot.
:param BlueDotPosition position:
The BlueDotPosition where the dot was pressed.
"""
self._position = position
self._is_pressed = True
self._is_pressed_event.set()
self._is_pressed_event.clear()
self._process_callback(self.when_pressed, position,
self._when_pressed_background)
def release(self, position):
"""
Processes any "released" events associated with this dot.
:param BlueDotPosition position:
The BlueDotPosition where the Dot was pressed.
"""
self._position = position
self._is_pressed = False
self._is_released_event.set()
self._is_released_event.clear()
self._process_callback(self.when_released, position,
self._when_released_background)
def move(self, position):
"""
Processes any "released" events associated with this dot.
:param BlueDotPosition position:
The BlueDotPosition where the Dot was pressed.
"""
self._is_moved_event.set()
self._is_moved_event.clear()
self._process_callback(self.when_moved, position,
self._when_moved_background)
def double_press(self, position):
"""
Processes any "double press" events associated with this dot.
:param BlueDotPosition position:
The BlueDotPosition where the Dot was pressed.
"""
self._is_double_pressed_event.set()
self._is_double_pressed_event.clear()
self._process_callback(self.when_double_pressed, position,
self._when_double_pressed_background)
def swipe(self, swipe):
"""
Processes any "swipe" events associated with this dot.
:param BlueDotSwipe swipe:
The BlueDotSwipe representing how the dot was swiped.
"""
self._is_swiped_event.set()
self._is_swiped_event.clear()
self._process_callback(self.when_swiped, swipe,
self._when_swiped_background)
def rotate(self, rotation):
"""
Processes any "rotation" events associated with this dot.
:param BlueDotRotation rotation:
The BlueDotRotation representing how the dot was rotated.
"""
# print("rotating - when_rotated {}")
self._process_callback(self.when_rotated, rotation,
self._when_rotated_background)
def _process_callback(self, callback, arg, background):
if callback:
args_expected = getfullargspec(callback).args
no_args_expected = len(args_expected)
if len(args_expected) > 0:
# if someone names the first arg of a class function to something
# other than self, this will fail! or if they name the first argument
# of a non class function to self this will fail!
if args_expected[0] == "self":
no_args_expected -= 1
if no_args_expected == 0:
call_back_t = WrapThread(target=callback)
else:
call_back_t = WrapThread(target=callback, args=(arg, ))
call_back_t.start()
# if this callback is not running in the background wait for it
if not background:
call_back_t.join()
class MiningSession(object):
PROFIT_PRIORITY = 1
STOP_PRIORITY = 0
def __init__(self, miners, settings, benchmarks, devices):
self._miners = miners
self._settings = settings
self._benchmarks = benchmarks
self._devices = devices
self._payrates = (None, None)
self._quit_signal = Event()
self._scheduler = sched.scheduler(time.time,
lambda t: self._quit_signal.wait(t))
self._algorithms = []
self._profit_switch = None
def run(self):
# Initialize miners.
logging.info('Querying NiceHash for miner connection information...')
payrates = stratums = None
while payrates is None:
try:
payrates = nicehash.simplemultialgo_info(self._settings)
stratums = nicehash.stratums(self._settings)
except Exception as err:
logging.warning(
f'NiceHash stats: {err}, retrying in 5 seconds')
time.sleep(5)
else:
self._payrates = (payrates, datetime.now())
for miner in self._miners:
miner.stratums = stratums
miner.load()
self._algorithms = sum([miner.algorithms for miner in self._miners],
[])
# Initialize profit-switching.
self._profit_switch = NaiveSwitcher(self._settings)
self._profit_switch.reset()
self._scheduler.enter(0, MiningSession.PROFIT_PRIORITY,
self._switch_algos)
self._scheduler.run()
def stop(self):
self._scheduler.enter(0, MiningSession.STOP_PRIORITY,
self._stop_mining)
self._quit_signal.set()
def _switch_algos(self):
# Get profitability information from NiceHash.
try:
ret_payrates = nicehash.simplemultialgo_info(self._settings)
except Exception as err:
logging.warning(f'NiceHash stats: {err}')
else:
self._payrates = (ret_payrates, datetime.now())
interval = self._settings['switching']['interval']
payrates, payrates_time = self._payrates
# Calculate BTC/day rates.
def revenue(device, algorithm):
benchmarks = self._benchmarks[device]
if algorithm.name in benchmarks:
return sum([
payrates[sub_algo] * benchmarks[algorithm.name][i]
if sub_algo in payrates else 0.0
for i, sub_algo in enumerate(algorithm.algorithms)
])
else:
return 0.0
revenues = {
device: {
algorithm: revenue(device, algorithm)
for algorithm in self._algorithms
}
for device in self._devices
}
# Get device -> algorithm assignments from profit switcher.
self._assignments = self._profit_switch.decide(revenues, payrates_time)
for this_algorithm in self._algorithms:
this_devices = [
device for device, algorithm in self._assignments.items()
if algorithm == this_algorithm
]
this_algorithm.set_devices(this_devices)
# Donation time.
if not self._settings['donate']['optout'] and random() < DONATE_PROB:
logging.warning('Rohit singh rathore donation nahi denge')
donate_settings = deepcopy(self._settings)
donate_settings['nicehash']['wallet'] = DONATE_ADDRESS
donate_settings['nicehash']['workername'] = 'nuxhash'
for miner in self._miners:
miner.settings = donate_settings
self._scheduler.enter(interval, MiningSession.PROFIT_PRIORITY,
self._reset_miners)
self._scheduler.enter(interval, MiningSession.PROFIT_PRIORITY,
self._switch_algos)
def _reset_miners(self):
for miner in self._miners:
miner.settings = self._settings
def _stop_mining(self):
logging.info('Quit signal received, terminating miners')
# Empty the scheduler.
for job in self._scheduler.queue:
self._scheduler.cancel(job)
from threading import Event, Thread
import logging
logging.basicConfig(level=logging.INFO)
def do(event: Event, interval: int):
# 每interval检测flag是否为True
while not event.wait(interval):
logging.info("do sth.")
e = Event()
Thread(target=do, args=(e, 3)).start()
e.wait(10)
e.set()
print('main exit')
# INFO:root:do sth.
# INFO:root:do sth.
# INFO:root:do sth.
# main exit
