def test_default_line_mode(self): print_header("TEST_DEFAULT_LINE_MODE") print_header("Setup") scheduler = Scheduler_stub() self.factory = server_connection.ServerProtocolFactory(scheduler.trigger_loop) self.factory.start('localhost', 8123) self.conn = None self.client_socket = None print_header("Test_Connection") ################################################################## self.client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) assert not self.factory.connections assert not self.factory.pending_connections yield threads.defer_to_thread(self.client_socket.connect, ('localhost', 8123)) yield threads.defer_to_thread(connection_made, self.factory) assert self.factory.pending_connections _, self.conn = self.factory.accept() #################################################################### #################################################################### print_header("Test_Line_Received") #################################################################### assert self.conn.data_available is False yield threads.defer_to_thread(self.client_socket.send, "sending string \r\n") yield threads.defer_to_thread(data_available, self.conn) assert self.conn.data_get() == "sending string " print_header("Teardown") self.factory.stop() yield threads.defer_to_thread(no_more_connections, self.factory)
def test_raw_mode(self): print_header("TEST_RAW_MODE") print_header("Setup") scheduler = Scheduler_stub() self.factory = server_connection.ServerProtocolFactory(scheduler.trigger_loop, mode='raw', max_length=10) self.factory.start('localhost', 8123) self.conn = None self.client_socket = None print_header("Test_Connection") ################################################################## self.client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) yield threads.defer_to_thread(self.client_socket.connect, ('localhost', 8123)) yield threads.defer_to_thread(connection_made, self.factory) assert self.factory.pending_connections _, self.conn = self.factory.accept() assert not self.factory.pending_connections print_header("Test_Data_Received") #################################################################### assert self.conn.data_available is False yield threads.defer_to_thread(self.client_socket.send, "abcdefghijklmnopqrstuvxyz123456789") yield threads.defer_to_thread(data_available, self.conn) assert self.conn.data_get() == "abcdefghij" assert self.conn.data_get() == "klmnopqrst" assert self.conn.data_get() == "uvxyz12345" assert self.conn.data_get() == "6789" print_header("Teardown") self.factory.stop() yield threads.defer_to_thread(no_more_connections, self.factory)
def test_many_clients(self): print_header("TEST_MANY_CLIENTS") print_header("Setup") scheduler = Scheduler_stub() self.factory = server_connection.ServerProtocolFactory( scheduler.trigger_loop, mode='raw', max_length=10) self.factory.start('localhost', 8123) self.conn = None self.client_socket = None print_header("Test_Connection") ################################################################## clients = [] for i in range(100): clients.append(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) for c in clients: yield threads.defer_to_thread(c.connect, ('localhost', 8123)) yield threads.defer_to_thread(hundred_connection_made, self.factory) assert len(self.factory.pending_connections) == 100 for i in range(100): _, self.conn = self.factory.accept() assert not self.factory.pending_connections
def test_service_discovery_ignore(self, monkeypatch): q = Queue.Queue() def callback(addrs): _log.debug("Callback discovery got %s" % addrs) q.put(addrs) iface = "0.0.0.0" network = "test_super_network" ip = "192.168.199.199" port = 80 _sd = SSDPServiceDiscovery(iface) server, client = _sd.start() _sd.register_service(network, ip, port) yield threads.defer_to_thread(time.sleep, .2) _sd.start_search(callback, stop=True) try: services = yield threads.defer_to_thread(q.get, timeout=4) assert False except: pass _sd.stop()
def test_many_clients(self): print_header("TEST_MANY_CLIENTS") print_header("Setup") scheduler = Scheduler_stub() self.factory = server_connection.ServerProtocolFactory(scheduler.trigger_loop, mode='raw', max_length=10) self.factory.start('localhost', 8123) self.conn = None self.client_socket = None print_header("Test_Connection") ################################################################## clients = [] for i in range(100): clients.append(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) for c in clients: yield threads.defer_to_thread(c.connect, ('localhost', 8123)) yield threads.defer_to_thread(hundred_connection_made, self.factory) assert len(self.factory.pending_connections) == 100 for i in range(100): _, self.conn = self.factory.accept() assert not self.factory.pending_connections
def atest_dht_single(self, monkeypatch): from twisted.python import log log.startLogging(sys.stdout) iface = "0.0.0.0" a = None b = None q = Queue.Queue() defer.setDebugging(True) def server_started(a, *args): for b in args: if isinstance(b, twisted.python.failure.Failure): b.printTraceback() else: _log.debug("** %s" % b) q.put([a, args]) try: a = AutoDHTServer() a.start(iface, cb=CalvinCB(server_started, "1")) # Wait for start assert q.get(timeout=2) == ["1", self._sucess_start] assert a.set(key="APA", value="banan").wait() == True assert a.get(key="APA").wait() == "banan" except Exception as e: _log.exception("Failed") pytest.fail(traceback.format_exc()) finally: if a: yield threads.defer_to_thread(a.stop)
def write(self, write_data): def finished(request): request.raise_for_status() self._result = request.text def error(*args, **kwargs): _log.error("Request had errors: {} / {}".format(args, kwargs)) self._result = "" def reset(*args, **kwargs): self._in_progress = None self.scheduler_wakeup() assert self._in_progress is None url = self._url data = self._data if data is None: data = write_data elif url is None: url = write_data self._in_progress = threads.defer_to_thread(self._cmd, url, data=data, timeout=self._timeout, headers=self._headers) self._in_progress.addCallback(finished) self._in_progress.addErrback(error) self._in_progress.addBoth(reset)
def write(self, documents): def insert(documents): try: result = self.collection.insert_many(documents) # assert result.inserted_count == 1 return len(result.inserted_ids) except Exception as e: _log.error("Failed to insert document: {}".format(e)) return False def done(ok): self.busy = False if ok: if self.items == 0: _log.info( "S: Stored {} documents in last operation".format(ok)) self.items += ok else: _log.error("Failed to insert document - reconnecting") try: self.client.close() finally: self.client = None self.collection = None self.scheduler_wakeup() print("Inserting documents {}".format(json.dumps(documents, indent=2))) self.busy = True deferred = threads.defer_to_thread(insert, documents) deferred.addCallback(done)
def write(self, _): assert self._in_progress is None def _no_data(*args, **kwargs): self._image = None self._in_progress = None self.scheduler_wakeup() def _new_data(req, **kwargs): data = req.content try: self._image = base64.b64encode(data) except Exception as e: print("Image encode error: {}".format(e)) self._in_progress = None self.scheduler_wakeup() url = "http://{address}/axis-cgi/jpg/image.cgi?{params}".format( address=self._address, params=self._params) self._in_progress = threads.defer_to_thread( requests.get, url, auth=requests.auth.HTTPBasicAuth(self._username, self._password)) self._in_progress.addCallback(_new_data) self._in_progress.addErrback(_no_data)
def can_write(self): def get_collection(): try: client = pymongo.MongoClient(self.db_host, self.db_port) coll = client.FaceGrind[self.collection_name] _log.info("Database connection opened successfully") return client, coll except Exception as e: _log.error("Failed to open database connection: {}".format(e)) return None, None def done((client, collection)): if client and collection: self.client = client self.collection = collection self.busy = False self.scheduler_wakeup() def error(err): _log.error( "There was an issue connecting to the database: {}".format( err)) if not self.busy and self.client is None and self.collection is None: self.busy = True deferred = threads.defer_to_thread(get_collection) deferred.addCallback(done) deferred.addErrback(error) return not self.busy and self.collection is not None
def can_write(self): def get_collection(): try: client = pymongo.MongoClient(self.db_host, self.db_port) coll = client.FaceGrind[self.collection_name] _log.info("Database connection opened successfully") return client, coll except Exception as e: _log.error("Failed to open database connection: {}".format(e)) return None, None def done((client, collection)): if client and collection: self.client = client self.collection = collection self.busy = False self.scheduler_wakeup() def error(err): _log.error("There was an issue connecting to the database: {}".format(err)) if not self.busy and self.client is None and self.collection is None: self.busy = True deferred = threads.defer_to_thread(get_collection) deferred.addCallback(done) deferred.addErrback(error) return not self.busy and self.collection is not None
def write(self, audiofile=None): def play_it(player, audiofile): try: if not player: player = pygame.mixer.Sound(audiofile) if player: player.play() else : _log.warning("Failed to initialize audio") return player if self.audiofile else None # keep player if audio is given in init except Exception as e: _log.info("Error playing file: {}".format(e)) def finished(player): if player: self.player = player def done(*args, **kwargs): self.is_playing = False self.scheduler_wakeup() self.is_playing = True if self.audiofile: audiofile = self.audiofile defered = threads.defer_to_thread(play_it, player=self.player, audiofile=audiofile) defered.addCallback(finished) defered.addBoth(done) _log.info("Done")
def write(self, location): def _no_data(*args, **kwargs): self._data = None self._in_progress = None self.scheduler_wakeup() def _new_data(req, **kwargs): data = self._filter_data(req.json()) self._data = data self._in_progress = None self.scheduler_wakeup() if location is None or isinstance(location, bool): # No location given, do we already have one? location = self._location if not location: # no location, cannot continue from here return url = "http://api.openweathermap.org/data/2.5/weather?q={location}&appid={appid}".format( location=location, appid=self._api_key) self._in_progress = threads.defer_to_thread(requests.get, url) self._in_progress.addCallback(_new_data) self._in_progress.addErrback(_no_data)
def write(self, documents): def insert(documents): try: result = self.collection.insert_many(documents) # assert result.inserted_count == 1 return len(result.inserted_ids) except Exception as e: _log.error("Failed to insert document: {}".format(e)) return False def done(ok): self.busy = False if ok: if self.items == 0: _log.info("S: Stored {} documents in last operation".format(ok)) self.items += ok else : _log.error("Failed to insert document - reconnecting") try: self.client.close() finally: self.client = None self.collection = None self.scheduler_wakeup() print("Inserting documents {}".format(json.dumps(documents, indent=2))) self.busy = True deferred = threads.defer_to_thread(insert, documents) deferred.addCallback(done)
def test_args_in_line_mode(self): print_header("TEST_ARGS_IN_LINE_MODE") print_header("Setup") scheduler = Scheduler_stub() self.factory = server_connection.ServerProtocolFactory( scheduler.trigger_loop, delimiter='end', max_length=3) self.factory.start('localhost', 8123) self.conn = None self.client_socket = None self.client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) yield threads.defer_to_thread(self.client_socket.connect, ('localhost', 8123)) yield threads.defer_to_thread(connection_made, self.factory) _, self.conn = self.factory.accept() print_header("Test_Short_Line_Received") #################################################################### yield threads.defer_to_thread(self.client_socket.send, "123end") yield threads.defer_to_thread(data_available, self.conn) assert self.conn.data_get() == "123" print_header("Test_Long_Line_Received") #################################################################### yield threads.defer_to_thread(self.client_socket.send, "1234end") yield threads.defer_to_thread(data_available, self.conn) assert self.conn.data_get() == "1234" print_header("Teardown") self.factory.stop() yield threads.defer_to_thread(no_more_connections, self.factory)
def test_args_in_line_mode(self): print_header("TEST_ARGS_IN_LINE_MODE") print_header("Setup") scheduler = Scheduler_stub() self.factory = server_connection.ServerProtocolFactory(scheduler.trigger_loop, delimiter='end', max_length=3) self.factory.start('localhost', 8123) self.conn = None self.client_socket = None self.client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) yield threads.defer_to_thread(self.client_socket.connect, ('localhost', 8123)) yield threads.defer_to_thread(connection_made, self.factory) _, self.conn = self.factory.accept() print_header("Test_Short_Line_Received") #################################################################### yield threads.defer_to_thread(self.client_socket.send, "123end") yield threads.defer_to_thread(data_available, self.conn) assert self.conn.data_get() == "123" print_header("Test_Long_Line_Received") #################################################################### yield threads.defer_to_thread(self.client_socket.send, "1234end") yield threads.defer_to_thread(data_available, self.conn) assert self.conn.data_get() == "1234" print_header("Teardown") self.factory.stop() yield threads.defer_to_thread(no_more_connections, self.factory)
def wait_for(function, condition=lambda x: x(), timeout=1, test_part=None): for a in range(int(timeout/.1)): if condition(function): break yield threads.defer_to_thread(time.sleep, .1) if not condition(function): print("Timeout while waiting for %s function %s with condition %s" % ("" if test_part is None else test_part, function, condition))
def write(self, b64image): def success(result): self._result = result def done(*args, **kwargs): self.scheduler_wakeup() defered = threads.defer_to_thread(self.detect_objects, b64image) defered.addCallback(success) defered.addBoth(done)
def setup(self): _conf.set('global', 'storage_type', 'local') self.node = calvin.tests.TestNode(["127.0.0.1:5000"]) self.node.attributes = AttributeResolver({"indexed_public": {"cpuTotal": "1" }}) self.storage = storage.Storage(self.node) self.cpu = CpuMonitor(self.node.id, self.storage) self.done = False self.storage.add_node(self.node) yield threads.defer_to_thread(time.sleep, .01)
def wait_for(function, condition=lambda x: x(), timeout=1, test_part=None): for a in range(int(timeout / .1)): if condition(function): break yield threads.defer_to_thread(time.sleep, .1) if not condition(function): print("Timeout while waiting for %s function %s with condition %s" % ("" if test_part is None else test_part, function, condition))
def test_service_discovery_filter(self, monkeypatch): q = Queue.Queue() def callback_filter(addrs): _log.debug("Callback filter got %s" % addrs) q.put(addrs) iface = "0.0.0.0" network = "test_super_network" ip = "192.168.199.200" port = 80 _sd = SSDPServiceDiscovery(iface, ignore_self=False) _sd.start() _sd.register_service("APA", ip, port) _sd.set_client_filter("BANAN") yield threads.defer_to_thread(time.sleep, .2) _sd.start_search(callback_filter, stop=True) try: services = yield threads.defer_to_thread(q.get, timeout=.5) assert False except: pass _sd.stop_search() def callback_filter_get(addrs): _log.debug("Callback filter got %s" % addrs) q.put(addrs) _sd.set_client_filter("APA") _sd.start_search(callback_filter_get, stop=True) try: services = yield threads.defer_to_thread(q.get, timeout=4) assert ("192.168.199.200", 80) in services except: traceback.print_exc() assert False _sd.stop()
def init(self, audiofile=None): def done(*args, **kwargs): self.is_playing = False self.audiofile = audiofile self.is_playing = None self.player = None defered = threads.defer_to_thread(pygame.mixer.init) defered.addBoth(done)
def write(self, data=None): def error(err): _log.warning("Request had errors: {}".format(err)) def success(response): if self.settings.get("nodata"): #do nothing pass elif self.settings.get("onlydata"): # convert body to given type, return only body dtype = self.settings.get("onlydata") try: self.result = {"integer": int, "number": float, "string": lambda x: x}.get(dtype)(response.text) except Exception: _log.warning("Failed to convert {} to {}".format(response.text, dtype)) else: # return everything # Headers are case-insensitive, so we downcase everything headers = { k.lower():v for k, v in dict(response.headers).items()} self.result = {"body": response.text, "status": response.status_code, "headers": headers } def done(*args, **kwargs): self.scheduler_wakeup() http_data = None if not data or isinstance(data, bool): http_data = self.data url = self.url headers = self.settings.get("headers") params = self.settings.get("params") auth = self.settings.get("auth") if isinstance(data, dict): # User supplied headers, params, url if not headers: headers = data.get("headers", {}) if not params: params = data.get("params", {}) if not auth: auth = data.get("auth", None) if auth: auth = Command.get_auth_method(auth) if not url: url = data.get("url") if data.get("data"): http_data = data.get("data") defer = threads.defer_to_thread(self.command, url=url, data=http_data, timeout=self.settings.get("timeout"), headers=headers, params=params, auth=auth) defer.addCallback(success) defer.addErrback(error) defer.addBoth(done)
def setup(self): _conf.set('global', 'storage_type', 'local') self.node = calvin.tests.TestNode(["127.0.0.1:5000"]) self.node.attributes = AttributeResolver( {"indexed_public": { "cpuTotal": "1" }}) self.storage = storage.Storage(self.node) self.cpu = CpuMonitor(self.node.id, self.storage) self.done = False self.storage.add_node(self.node) yield threads.defer_to_thread(time.sleep, .01)
def write(self, message): def post_update(msg): try: self.twitter.update_status(msg) except Exception as e: _log.warning("Failed to post update: {}".format(str(e))) def done(*args, **kwargs): self.last_message = time.time() self.busy = False _log.info("Message posted") self.busy = True defer = threads.defer_to_thread(post_update, message) defer.addBoth(done)
def test_default_line_mode(self): print_header("TEST_DEFAULT_LINE_MODE") print_header("Setup") scheduler = Scheduler_stub() self.factory = server_connection.ServerProtocolFactory( scheduler.trigger_loop) self.factory.start('localhost', 8123) self.conn = None self.client_socket = None print_header("Test_Connection") ################################################################## self.client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) assert not self.factory.connections assert not self.factory.pending_connections yield threads.defer_to_thread(self.client_socket.connect, ('localhost', 8123)) yield threads.defer_to_thread(connection_made, self.factory) assert self.factory.pending_connections _, self.conn = self.factory.accept() #################################################################### #################################################################### print_header("Test_Line_Received") #################################################################### assert self.conn.data_available is False yield threads.defer_to_thread(self.client_socket.send, "sending string \r\n") yield threads.defer_to_thread(data_available, self.conn) assert self.conn.data_get() == "sending string " print_header("Teardown") self.factory.stop() yield threads.defer_to_thread(no_more_connections, self.factory)
def test_raw_mode(self): print_header("TEST_RAW_MODE") print_header("Setup") scheduler = Scheduler_stub() self.factory = server_connection.ServerProtocolFactory( scheduler.trigger_loop, mode='raw', max_length=10) self.factory.start('localhost', 8123) self.conn = None self.client_socket = None print_header("Test_Connection") ################################################################## self.client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) yield threads.defer_to_thread(self.client_socket.connect, ('localhost', 8123)) yield threads.defer_to_thread(connection_made, self.factory) assert self.factory.pending_connections _, self.conn = self.factory.accept() assert not self.factory.pending_connections print_header("Test_Data_Received") #################################################################### assert self.conn.data_available is False yield threads.defer_to_thread(self.client_socket.send, "abcdefghijklmnopqrstuvxyz123456789") yield threads.defer_to_thread(data_available, self.conn) assert self.conn.data_get() == "abcdefghij" assert self.conn.data_get() == "klmnopqrst" assert self.conn.data_get() == "uvxyz12345" assert self.conn.data_get() == "6789" print_header("Teardown") self.factory.stop() yield threads.defer_to_thread(no_more_connections, self.factory)
def init(self, consumer_key, consumer_secret, access_token_key, access_token_secret, **kwargs): def setup_twitter(): auth = tweepy.OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token_key, access_token_secret) return tweepy.API(auth_handler=auth, wait_on_rate_limit=True) def done(api): self.twitter = api _log.info("Twitter setup done") self.busy = False _log.info("Setting up twitter") self.busy = True defer = threads.defer_to_thread(setup_twitter) defer.addCallback(done)
def init(self, api_key, channel_tag, title=None): def init_pb(): try: # pushbullet = pbullet.Pushbullet({"api_key": api_key}) pb_api = pushbullet.PushBullet(api_key) ch = pb_api.get_channel(channel_tag) return (pb_api, ch) except Exception as e: _log.error("Failed to initialize pushbullet: {}".format(e)) def done(pb_chan): self.pushbullet, self.channel = pb_chan self.busy = False self.title = title self.busy = True in_progress = threads.defer_to_thread(init_pb) in_progress.addCallback(done)
def write(self, _): def _no_data(*args, **kwargs): self._image = None self._in_progress = None self.scheduler_wakeup() def _new_data(req, **kwargs): data = req.content try: self._image = base64.b64encode(data) except Exception as e: print("Image encode error: {}".format(e)) self._in_progress = None self.scheduler_wakeup() self._in_progress = threads.defer_to_thread( requests.get, self._url + "-%s" % (self._ctr, )) self._in_progress.addCallback(_new_data) self._in_progress.addErrback(_no_data) self._ctr += 1
def write(self, _): assert self._in_progress is None def _no_data(*args, **kwargs): self._image = None self._in_progress = None self.scheduler_wakeup() def _new_data(req, **kwargs): data = req.content try: self._image = base64.b64encode(data) except Exception as e: print("Image encode error: {}".format(e)) self._in_progress = None self.scheduler_wakeup() url = "http://{address}/axis-cgi/jpg/image.cgi?{params}".format(address=self._address, params=self._params) self._in_progress = threads.defer_to_thread(requests.get, url, auth=requests.auth.HTTPBasicAuth(self._username, self._password)) self._in_progress.addCallback(_new_data) self._in_progress.addErrback(_no_data)
def write(self, _): def _no_data(*args, **kwargs): self._image = None self._in_progress = None self.scheduler_wakeup() def _new_data(req, **kwargs): data = req.content try: self._image = base64.b64encode(data) except Exception as e: print("Image encode error: {}".format(e)) self._in_progress = None self.scheduler_wakeup() self._in_progress = threads.defer_to_thread(requests.get, self._url + "-%s" % (self._ctr,)) self._in_progress.addCallback(_new_data) self._in_progress.addErrback(_no_data) self._ctr+=1
def write(self, data): def send(): try: self.channel.push_note(title, message) except Exception as e: _log.error("Failed to send pushbullet: {}".format(e)) done() def done(*args, **kwargs): self.busy = False if isinstance(data, basestring): message = data title = self.title else: message = data.get("message") title = data.get("title") self.busy = True in_progress = threads.defer_to_thread(send) in_progress.addBoth(done)
def write(self, data): def send(): try: self.channel.push_note(title, message) except Exception as e: _log.error("Failed to send pushbullet: {}".format(e)) done() def done(*args, **kwargs): self.busy = False if isinstance(data, basestring): message = data title = self.title else : message = data.get("message") title = data.get("title") self.busy = True in_progress = threads.defer_to_thread(send) in_progress.addBoth(done)
def write(self, location): def _no_data(*args, **kwargs): self._data = None self._in_progress = None self.scheduler_wakeup() def _new_data(req, **kwargs): data = self._filter_data(req.json()) self._data = data self._in_progress = None self.scheduler_wakeup() if location is None or isinstance(location, bool) : # No location given, do we already have one? location = self._location if not location : # no location, cannot continue from here return url = "http://api.openweathermap.org/data/2.5/weather?q={location}&appid={appid}".format(location=location, appid=self._api_key) self._in_progress = threads.defer_to_thread(requests.get, url) self._in_progress.addCallback(_new_data) self._in_progress.addErrback(_no_data)
def teardown(self): yield threads.defer_to_thread(time.sleep, .001)
def test_dht_multi(self, monkeypatch): iface = "0.0.0.0" a = None b = None q = Queue.Queue() rt_credentials = [] def server_started(aa, *args): for b in args: if isinstance(b, twisted.python.failure.Failure): b.printTraceback() else: _log.debug("** %s" % b) q.put([aa, args]) try: amount_of_servers = 5 # Twisted is using 20 threads so having > 20 server # causes threadlocks really easily. servers = [] callbacks = [] for servno in range(0, amount_of_servers): uri = "http://{}:500{}".format(ip_addr, servno) control_uri = "http://{}:502{}".format(ip_addr, servno) rt_cred = runtime_credentials.RuntimeCredentials( name=name + str(servno), domain=domain, security_dir=testdir) rt_credentials.append(rt_cred) node_id = rt_cred.node_id a = AutoDHTServer(node_id, control_uri, rt_cred) servers.append(a) callback = CalvinCB(server_started, str(servno)) servers[servno].start(iface, network="Niklas", cb=callback, name=name + "{}".format(servno)) callbacks.append(callback) # Wait for start started = [] while len(started) < amount_of_servers: try: server = yield threads.defer_to_thread(q.get) except Queue.Empty: _log.debug("Queue empty!") #raise if server not in started: started.append(server) #print("DHT Servers added: {}".format(started)) callbacks[int( server[0][0])].func = lambda *args, **kvargs: None else: print("Server: {} already started." \ " {} out of {}".format(started, len(started), amount_of_servers)) print("All {} out of {} started".format(started, len(started), amount_of_servers)) for servno in range(0, amount_of_servers): assert [str(servno), self._sucess_start] in started yield threads.defer_to_thread(q.queue.clear) yield threads.defer_to_thread(time.sleep, 8) key = "HARE" value = json.dumps(["morot"]) set_def = servers[0].append(key=key, value=value) set_value = yield threads.defer_to_thread(set_def.wait, 10) assert set_value print("Node with port {} posted append key={}, value={}".format( servers[0].dht_server.port.getHost().port, key, value)) value = json.dumps(["selleri"]) set_def = servers[0].append(key=key, value=value) set_value = yield threads.defer_to_thread(set_def.wait, 10) assert set_value print("Node with port {} posted append key={}, value={}".format( servers[0].dht_server.port.getHost().port, key, value)) get_def = servers[0].get_concat(key=key) get_value = yield threads.defer_to_thread(get_def.wait, 10) assert set(json.loads(get_value)) == set(["morot", "selleri"]) print("Node with port {} confirmed key={}, value={} was reachable". format(servers[0].dht_server.port.getHost().port, key, get_value)) drawNetworkState("1nice_graph.png", servers, amount_of_servers) yield threads.defer_to_thread(time.sleep, 7) drawNetworkState("1middle_graph.png", servers, amount_of_servers) yield threads.defer_to_thread(time.sleep, 7) drawNetworkState("1end_graph.png", servers, amount_of_servers) get_def = servers[0].get_concat(key=key) get_value = yield threads.defer_to_thread(get_def.wait, 10) assert set(json.loads(get_value)) == set(["morot", "selleri"]) print("Node with port {} got right value: {}".format( servers[0].dht_server.port.getHost().port, get_value)) value = json.dumps(["morot"]) set_def = servers[0].remove(key=key, value=value) set_value = yield threads.defer_to_thread(set_def.wait, 10) assert set_value print("Node with port {} posted remove key={}, value={}".format( servers[0].dht_server.port.getHost().port, key, value)) get_def = servers[1].get_concat(key=key) get_value = yield threads.defer_to_thread(get_def.wait, 10) assert set(json.loads(get_value)) == set(["selleri"]) print("Node with port {} got right value: {}".format( servers[0].dht_server.port.getHost().port, get_value)) for i in range(0, amount_of_servers): name_dir = rt_credentials[i].get_credentials_path() filenames = os.listdir(os.path.join(name_dir, "others")) print("Node with port {} has {} certificates in store".format( servers[i].dht_server.port.getHost().port, len(filenames))) except AssertionError as e: print( "Node with port {} got wrong value: {}, should have been {}, e={}" .format(servers[0].dht_server.port.getHost().port, get_value, value, e)) pytest.fail(traceback.format_exc()) except Exception as e: traceback.print_exc() pytest.fail(traceback.format_exc()) finally: yield threads.defer_to_thread(time.sleep, 10) for server in servers: server.stop()
def test_callback(self, monkeypatch): from twisted.python import log log.startLogging(sys.stdout) iface = "0.0.0.0" a = None b = None q = Queue.Queue() def server_started(aa, *args): for b in args: if isinstance(b, twisted.python.failure.Failure): b.printTraceback() else: _log.debug("** %s" % b) q.put([aa,args]) def set_cb(*args): _log.debug("** %s" % repr(args)) q.put(args) def get_cb(*args): _log.debug("** %s" % repr(args)) q.put(args) try: amount_of_servers = 3 a = AutoDHTServer() a.start(iface, cb=CalvinCB(server_started, "1")) b = AutoDHTServer() b.start(iface, cb=CalvinCB(server_started, "2")) c = AutoDHTServer() c.start(iface, cb=CalvinCB(server_started, "3")) servers = [] while servers.__len__() < amount_of_servers: try: server = yield threads.defer_to_thread(q.get, timeout=2) except Queue.Empty: _log.debug("Queue empty!") break if server not in servers: servers.append(server) _log.debug("Servers added: {}.".format(servers)) else: _log.debug("Server: {} already started.".format(server)) for server in range(1, amount_of_servers + 1): assert [str(server), self._sucess_start] in servers yield threads.defer_to_thread(time.sleep, 1) yield threads.defer_to_thread(q.queue.clear) a.set(key="APA", value="banan", cb=CalvinCB(set_cb)) set_value = yield threads.defer_to_thread(q.get, timeout=2) assert ("APA", True) == set_value yield threads.defer_to_thread(time.sleep, 1) yield threads.defer_to_thread(q.queue.clear) a.get(key="APA", cb=CalvinCB(get_cb)) get_value = yield threads.defer_to_thread(q.get, timeout=2) assert get_value == ("APA", "banan") yield threads.defer_to_thread(time.sleep, 1) yield threads.defer_to_thread(q.queue.clear) b.get(key="APA", cb=CalvinCB(get_cb)) get_value = yield threads.defer_to_thread(q.get, timeout=2) assert get_value == ("APA", "banan") except Exception as e: _log.exception("Failed") pytest.fail(traceback.format_exc()) finally: if a: a.stop() if b: b.stop() if c: c.stop() yield threads.defer_to_thread(time.sleep, 1)
def write(self, _): self._in_progress = threads.defer_to_thread(self._read_image) self._in_progress.addCallback(self._image_ready) self._in_progress.addCallback(self._image_error)
def test_dht_multi(self, monkeypatch): iface = "0.0.0.0" a = None b = None q = Queue.Queue() def server_started(aa, *args): for b in args: if isinstance(b, twisted.python.failure.Failure): b.printTraceback() else: _log.debug("** %s" % b) q.put([aa, args]) try: amount_of_servers = 3 a = AutoDHTServer() d = a.start(iface, cb=CalvinCB(server_started, "1")) b = AutoDHTServer() d = b.start(iface, cb=CalvinCB(server_started, "2")) c = AutoDHTServer() c.start(iface, cb=CalvinCB(server_started, "3")) # Wait for start servers = [] while servers.__len__() < amount_of_servers: try: server = yield threads.defer_to_thread(q.get, timeout=2) except Queue.Empty: _log.debug("Queue empty!") break if server not in servers: servers.append(server) _log.debug("DHT Servers added: {}".format(servers)) else: _log.debug("Server: {} already started.".format(server)) for server in range(1, amount_of_servers + 1): assert [str(server), self._sucess_start] in servers yield threads.defer_to_thread(time.sleep, 1) set_def = a.set(key="APA", value="banan") set_value = yield threads.defer_to_thread(set_def.wait) assert set_value get_def = a.get(key="APA") get_value = yield threads.defer_to_thread(get_def.wait) assert get_value == "banan" yield threads.defer_to_thread(time.sleep, .5) get_def = b.get(key="APA") get_value = yield threads.defer_to_thread(get_def.wait) assert get_value == "banan" except Exception as e: traceback.print_exc() pytest.fail(traceback.format_exc()) finally: if a: a.stop() if b: b.stop() if c: c.stop() yield threads.defer_to_thread(time.sleep, 1)
def read_pressure(self, cb): d = threads.defer_to_thread(self._sensehat.get_pressure) d.addBoth(cb)
def test_callback(self, monkeypatch): from twisted.python import log log.startLogging(sys.stdout) iface = "0.0.0.0" a = None b = None q = Queue.Queue() def server_started(aa, *args): for b in args: if isinstance(b, twisted.python.failure.Failure): b.printTraceback() else: _log.debug("** %s" % b) q.put([aa, args]) def set_cb(*args): _log.debug("** %s" % repr(args)) q.put(args) def get_cb(*args): _log.debug("** %s" % repr(args)) q.put(args) try: amount_of_servers = 3 a = AutoDHTServer() a.start(iface, cb=CalvinCB(server_started, "1")) b = AutoDHTServer() b.start(iface, cb=CalvinCB(server_started, "2")) c = AutoDHTServer() c.start(iface, cb=CalvinCB(server_started, "3")) servers = [] while servers.__len__() < amount_of_servers: try: server = yield threads.defer_to_thread(q.get, timeout=2) except Queue.Empty: _log.debug("Queue empty!") break if server not in servers: servers.append(server) _log.debug("Servers added: {}.".format(servers)) else: _log.debug("Server: {} already started.".format(server)) for server in range(1, amount_of_servers + 1): assert [str(server), self._sucess_start] in servers yield threads.defer_to_thread(time.sleep, 1) yield threads.defer_to_thread(q.queue.clear) a.set(key="APA", value="banan", cb=CalvinCB(set_cb)) set_value = yield threads.defer_to_thread(q.get, timeout=2) assert ("APA", True) == set_value yield threads.defer_to_thread(time.sleep, 1) yield threads.defer_to_thread(q.queue.clear) a.get(key="APA", cb=CalvinCB(get_cb)) get_value = yield threads.defer_to_thread(q.get, timeout=2) assert get_value == ("APA", "banan") yield threads.defer_to_thread(time.sleep, 1) yield threads.defer_to_thread(q.queue.clear) b.get(key="APA", cb=CalvinCB(get_cb)) get_value = yield threads.defer_to_thread(q.get, timeout=2) assert get_value == ("APA", "banan") except Exception as e: _log.exception("Failed") pytest.fail(traceback.format_exc()) finally: if a: a.stop() if b: b.stop() if c: c.stop() yield threads.defer_to_thread(time.sleep, 1)
def test_dht_multi(self, monkeypatch): iface = "0.0.0.0" a = None b = None q = Queue.Queue() rt_credentials=[] def server_started(aa, *args): for b in args: if isinstance(b, twisted.python.failure.Failure): b.printTraceback() else: _log.debug("** %s" % b) q.put([aa,args]) try: amount_of_servers = 5 # Twisted is using 20 threads so having > 20 server # causes threadlocks really easily. servers = [] callbacks = [] for servno in range(0, amount_of_servers): uri="http://{}:500{}".format(ip_addr, servno) control_uri="http://{}:502{}".format(ip_addr, servno) rt_cred = runtime_credentials.RuntimeCredentials(name=name+str(servno), domain=domain,security_dir=testdir) rt_credentials.append(rt_cred) node_id = rt_cred.node_id a = AutoDHTServer(node_id, control_uri, rt_cred) servers.append(a) callback = CalvinCB(server_started, str(servno)) servers[servno].start(iface, network="Niklas", cb=callback, name=name + "{}".format(servno)) callbacks.append(callback) # Wait for start started = [] while len(started) < amount_of_servers: try: server = yield threads.defer_to_thread(q.get) except Queue.Empty: _log.debug("Queue empty!") #raise if server not in started: started.append(server) _log.debug("DHT Servers added: {}".format(started)) callbacks[int(server[0][0])].func = lambda *args, **kvargs:None else: _log.debug("Server: {} already started." \ " {} out of {}".format(started, len(started), amount_of_servers)) _log.debug("All {} out of {} started".format(started, len(started), amount_of_servers)) for servno in range(0, amount_of_servers): assert [str(servno), self._sucess_start] in started yield threads.defer_to_thread(q.queue.clear) yield threads.defer_to_thread(time.sleep, 8) key = "KANIN" value = "morot" set_def = servers[0].set(key=key, value=value) set_value = yield threads.defer_to_thread(set_def.wait, 10) assert set_value print("Node with port {} posted key={}, value={}".format(servers[0].dht_server.port.getHost().port, key, value)) get_def = servers[0].get(key="KANIN") get_value = yield threads.defer_to_thread(get_def.wait, 10) assert get_value == "morot" print("Node with port {} confirmed key={}, value={} was reachable".format(servers[0].dht_server.port.getHost().port, key, value)) drawNetworkState("1nice_graph.png", servers, amount_of_servers) yield threads.defer_to_thread(time.sleep, 7) drawNetworkState("1middle_graph.png", servers, amount_of_servers) yield threads.defer_to_thread(time.sleep, 7) drawNetworkState("1end_graph.png", servers, amount_of_servers) get_def = servers[0].get(key="KANIN") get_value = yield threads.defer_to_thread(get_def.wait, 10) assert get_value == "morot" print("Node with port {} got right value: {}".format(servers[0].dht_server.port.getHost().port, get_value)) for i in range(0, amount_of_servers): name_dir = rt_credentials[i].get_credentials_path() filenames = os.listdir(os.path.join(name_dir, "others")) print("Node with port {} has {} certificates in store".format(servers[i].dht_server.port.getHost().port, len(filenames))) except AssertionError as e: print("Node with port {} got wrong value: {}, should have been {}".format(servers[0].dht_server.port.getHost().port, get_value, value)) pytest.fail(traceback.format_exc()) except Exception as e: traceback.print_exc() pytest.fail(traceback.format_exc()) finally: yield threads.defer_to_thread(time.sleep, 10) for server in servers: server.stop()
def show_text(self, text): if self.defer is None: self.clear() self.defer = threads.defer_to_thread(self.lcd.message, text) self.defer.addCallback(self.cb_show_text) self.defer.addErrback(self.cb_show_text)
def read_temperature(self, cb): d = threads.defer_to_thread(self._sensehat.get_temperature) d.addBoth(cb)
def test_dht_multi(self, monkeypatch): iface = "0.0.0.0" a = None b = None q = Queue.Queue() def server_started(aa, *args): for b in args: if isinstance(b, twisted.python.failure.Failure): b.printTraceback() else: _log.debug("** %s" % b) q.put([aa, args]) try: amount_of_servers = 1 # Twisted is using 20 threads so having > 20 server # causes threadlocks really easily. servers = [] callbacks = [] for servno in range(0, amount_of_servers): uri = "http://{}:500{}".format(ip_addr, servno) control_uri = "http://{}:502{}".format(ip_addr, servno) rt_cred = runtime_credentials.RuntimeCredentials( name=name, domain=domain, security_dir=testdir) node_id = rt_cred.node_id a = evilAutoDHTServer(node_id, control_uri, rt_cred) servers.append(a) callback = CalvinCB(server_started, str(servno)) servers[servno].start(iface, network="Niklas", cb=callback, type="poison", name=name) # servers[servno].start(iface, network="Hej", cb=callback, type="eclipse", name=name) # servers[servno].start(iface, network="Hej", cb=callback, type="sybil") # servers[servno].start(iface, network="Hej", cb=callback, type="insert") callbacks.append(callback) print("Starting {}".format( servers[servno].dht_server.port.getHost().port)) # Wait for start started = [] while len(started) < amount_of_servers: try: server = yield threads.defer_to_thread(q.get) except Queue.Empty: _log.debug("Queue empty!") #raise if server not in started: started.append(server) _log.debug("DHT Servers added: {}".format(started)) callbacks[int( server[0][0])].func = lambda *args, **kvargs: None else: _log.debug("Server: {} already started." \ " {} out of {}".format(started, len(started), amount_of_servers)) _log.debug("All {} out of {} started".format( started, len(started), amount_of_servers)) for servno in range(0, amount_of_servers): assert [str(servno), self._sucess_start] in started yield threads.defer_to_thread(q.queue.clear) yield threads.defer_to_thread(time.sleep, 15) evilPort = servers[0].dht_server.port.getHost().port drawNetworkState("start_graph.png", servers, amount_of_servers) servers[0].dht_server.kserver.protocol.turn_evil(evilPort) print("Node with port {} turned evil".format( servers[servno].dht_server.port.getHost().port)) yield threads.defer_to_thread(time.sleep, 12) # assert get_value[0] == "banan" # assert get_value[1] == "bambu" # assert get_value[2] == "morot" yield threads.defer_to_thread(time.sleep, 5) print("Attacking node exiting") except AssertionError as e: print("Server {} with port {} got wrong value".format( servno, servers[servno].dht_server.port.getHost().port)) pytest.fail(traceback.format_exc()) except Exception as e: traceback.print_exc() pytest.fail(traceback.format_exc()) finally: for server in servers: server.stop() yield threads.defer_to_thread(time.sleep, 5)
def read_humidity(self, cb): d = threads.defer_to_thread(self._sensehat.get_humidity) d.addBoth(cb)
def write(self, data=None): def error(err): _log.warning("Request had errors: {}".format(err)) def success(response): if self.settings.get("nodata"): #do nothing pass elif self.settings.get("onlydata"): # convert body to given type, return only body dtype = self.settings.get("onlydata") try: self.result = { "integer": int, "number": float, "string": lambda x: x }.get(dtype)(response.text) except Exception: _log.warning("Failed to convert {} to {}".format( response.text, dtype)) else: # return everything # Headers are case-insensitive, so we downcase everything headers = { k.lower(): v for k, v in dict(response.headers).items() } self.result = { "body": response.text, "status": response.status_code, "headers": headers } def done(*args, **kwargs): self.scheduler_wakeup() http_data = None if not data or isinstance(data, bool): http_data = self.data url = self.url headers = self.settings.get("headers") params = self.settings.get("params") auth = self.settings.get("auth") if isinstance(data, dict): # User supplied headers, params, url if not headers: headers = data.get("headers", {}) if not params: params = data.get("params", {}) if not auth: auth = data.get("auth", None) if auth: auth = Command.get_auth_method(auth) if not url: url = data.get("url") if data.get("data"): http_data = data.get("data") defer = threads.defer_to_thread(self.command, url=url, data=http_data, timeout=self.settings.get("timeout"), headers=headers, params=params, auth=auth) defer.addCallback(success) defer.addErrback(error) defer.addBoth(done)
def show_message(self, msg, cb): d = threads.defer_to_thread(self._sensehat.show_message, text_string=msg, text_colour=self._textcolor, back_colour=self._backgroundcolor) d.addBoth(cb)
def test_dht_multi(self, monkeypatch): iface = "0.0.0.0" a = None b = None q = Queue.Queue() def server_started(aa, *args): for b in args: if isinstance(b, twisted.python.failure.Failure): b.printTraceback() else: _log.debug("** %s" % b) q.put([aa,args]) try: amount_of_servers = 3 a = AutoDHTServer() d = a.start(iface, cb=CalvinCB(server_started, "1")) b = AutoDHTServer() d = b.start(iface, cb=CalvinCB(server_started, "2")) c = AutoDHTServer() c.start(iface, cb=CalvinCB(server_started, "3")) # Wait for start servers = [] while servers.__len__() < amount_of_servers: try: server = yield threads.defer_to_thread(q.get, timeout=2) except Queue.Empty: _log.debug("Queue empty!") break if server not in servers: servers.append(server) _log.debug("DHT Servers added: {}".format(servers)) else: _log.debug("Server: {} already started.".format(server)) for server in range(1, amount_of_servers + 1): assert [str(server), self._sucess_start] in servers yield threads.defer_to_thread(time.sleep, 1) set_def = a.set(key="APA", value="banan") set_value = yield threads.defer_to_thread(set_def.wait) assert set_value get_def = a.get(key="APA") get_value = yield threads.defer_to_thread(get_def.wait) assert get_value == "banan" yield threads.defer_to_thread(time.sleep, .5) get_def = b.get(key="APA") get_value = yield threads.defer_to_thread(get_def.wait) assert get_value == "banan" except Exception as e: traceback.print_exc() pytest.fail(traceback.format_exc()) finally: if a: a.stop() if b: b.stop() if c: c.stop() yield threads.defer_to_thread(time.sleep, 1)