def truncator(): while len(clients) < N: Tasklet.sleep(1.0) Tasklet.sleep(5.0) while True: print 'connecting truncator' client = Socket.connect(('localhost', 6379)) stream = BufferedStream(client) print 'issueing trunc of append only log' bgrewriteaof(stream) stream.close() Tasklet.sleep(120.0)
def _send(self, identifier, body, timeout=30): try: conn = Socket.connect((self.host, self.port), timeout) try: req = u"POST / HTTP/1.1\r\nHost: %s\r\nContent-length: %d\r\nX-Realplexor: %s=%s%s\r\n\r\n%s" % (self.host, len(body), (self.identifier or ""), ("%s:%s@" % (self.login, self.password) if self.login else ""), identifier or "", body) if type(req) == unicode: req = req.encode("utf-8") reader = BufferedReader(conn, Buffer(1024)) writer = BufferedWriter(conn, Buffer(1024)) writer.write_bytes(req) writer.flush() conn.socket.shutdown(socket.SHUT_WR) response = "" while True: try: chunk = reader.read_bytes_available() except EOFError: chunk = None if not chunk: break response += chunk if response: m = re_split_headers.split(response, 1) if not m: raise RealplexorError("Non-HTTP response received:\n%s" % response) headers = m[0] body = m[1] m = re_http_status_line.match(headers) if not m: raise RealplexorError("Non-HTTP response received:\n%s" % response) status, code = m.group(1, 2) if code != "200": raise RealplexorError("Request failed: %s\n%s" % (status, body)) m = re_content_length.search(headers) if not m: raise RealplexorError("No Content-Length header in response headers:\n%s" % headers) expected_len = int(m.group(1)) if len(body) != expected_len: raise RealplexorError("Response length (%d) is different from one specified in the Content-Length header (%d): possibly broken response" % (len(body), expected_len)) return body return response finally: conn.close() except socket.error as e: raise RealplexorSocketError("Socket error: %s" % e) except socket.gaierror as e: raise RealplexorError("DNS error: %s" % e) except socket.timeout as e: raise RealplexorError("Timeout error: %s" % e) except IOError as e: raise RealplexorSocketError("IOError: %s" % e)
def whois(self, query, hostname, flags): """Perform initial lookup with TLD whois server then, if the quick flag is false, search that result for the region-specifc whois server and do a lookup there for contact details """ response = '' engine = QueryEngine() result = engine.asynchronous(hostname, adns.rr.ADDR) if len(result[3]): ips = [rr[1] for rr in result[3]] ip = random.choice(ips) try: s = Socket.connect((ip, 43)) writer = BufferedWriter(s, Buffer(1024)) reader = BufferedReader(s, Buffer(1024)) if (hostname == NICClient.GERMNICHOST): writer.write_bytes("-T dn,ace -C US-ASCII " + query + "\r\n") else: writer.write_bytes(query + "\r\n") writer.flush() while True: try: d = reader.read_bytes_available() except EOFError: break response += d if not d: break except IOError: pass nhost = None if (flags & NICClient.WHOIS_RECURSE and nhost == None): nhost = self.findwhois_server(response, hostname) if (nhost != None): response += self.whois(query, nhost, 0) return response
def connector(): for i in range(N): client = Socket.connect(('localhost', 6379)) clients.append(BufferedStream(client)) print i print 'connector done', N
def _get_socket(self, host, port, timeout): new_socket = Socket.connect((host, port)) self._reader = BufferedReader(new_socket, Buffer(1024)) self._writer = BufferedWriter(new_socket, Buffer(1024)) self.file = self._reader.file() return new_socket
def testHTTP10(self): s = Socket.connect(('localhost', SERVER_PORT)) r1 = self.fetch10(s, '/hello/1') r2 = self.fetch10(s, '/hello/2') self.assertEquals('', r2)
def connect(self): self._stream = BufferedStream(Socket.connect(self._address))
def testTextProtocol(self): from concurrence.io import Socket, BufferedStream from concurrence.memcache.protocol import MemcacheProtocol socket = Socket.connect((MEMCACHE_IP, 11212)) stream = BufferedStream(socket) writer = stream.writer reader = stream.reader try: protocol = MemcacheProtocol.create("textblaat") self.fail("expected error") except MemcacheError: pass protocol = MemcacheProtocol.create("text") self.assertTrue(protocol is MemcacheProtocol.create(protocol)) protocol.set_codec("raw") protocol.write_set(writer, 'hello', 'world', 0, 0) writer.flush() self.assertEquals((MemcacheResult.STORED, None), protocol.read_set(reader)) N = 100 for i in range(N): protocol.write_set(writer, 'test%d' % i, 'hello world %d' % i, 0, 0) writer.flush() self.assertEquals((MemcacheResult.STORED, None), protocol.read_set(reader)) #single get for i in range(N): protocol.write_get(writer, ['test%d' % i]) writer.flush() result = protocol.read_get(reader) self.assertEquals((MemcacheResult.OK, {'test%d' % i: 'hello world %d' % i}), result) #multi get for i in range(0, N, 10): keys = ['test%d' % x for x in range(i, i + 10)] protocol.write_get(writer, keys) writer.flush() result, values = protocol.read_get(reader) self.assertEquals(MemcacheResult.OK, result) self.assertEquals(10, len(values)) #multi get pipeline, e.g. write N gets, but don't read out the results yet for i in range(0, N, 10): keys = ['test%d' % x for x in range(i, i + 10)] protocol.write_get(writer, keys) writer.flush() #now read the results for i in range(0, N, 10): result, values = protocol.read_get(reader) self.assertEquals(10, len(values)) self.assertTrue(('test%d' % i) in values) #pipelined multiget with same set of keys protocol.write_get(writer, ['test2', 'test8', 'test9', 'test11', 'test23', 'test24', 'test29', 'test31', 'test34']) writer.flush() protocol.write_get(writer, ['test2', 'test8', 'test9', 'test11', 'test23', 'test24', 'test29', 'test31', 'test34']) writer.flush() result1 = protocol.read_get(reader) result2 = protocol.read_get(reader) self.assertEquals(result1, result2)
def testTextProtocol(self): from concurrence.io import Socket, BufferedStream from concurrence.memcache.protocol import MemcacheProtocol socket = Socket.connect((MEMCACHE_IP, 11211)) stream = BufferedStream(socket) writer = stream.writer reader = stream.reader try: protocol = MemcacheProtocol.create("textblaat") self.fail("expected error") except MemcacheError: pass protocol = MemcacheProtocol.create("text") self.assertTrue(protocol is MemcacheProtocol.create(protocol)) protocol.set_codec("raw") protocol.write_set(writer, 'hello', 'world', 0, 0) writer.flush() self.assertEquals((MemcacheResult.STORED, None), protocol.read_set(reader)) N = 100 for i in range(N): protocol.write_set(writer, 'test%d' % i, 'hello world %d' % i, 0, 0) writer.flush() self.assertEquals((MemcacheResult.STORED, None), protocol.read_set(reader)) #single get for i in range(N): protocol.write_get(writer, ['test%d' % i]) writer.flush() result = protocol.read_get(reader) self.assertEquals((MemcacheResult.OK, { 'test%d' % i: 'hello world %d' % i }), result) #multi get for i in range(0, N, 10): keys = ['test%d' % x for x in range(i, i + 10)] protocol.write_get(writer, keys) writer.flush() result, values = protocol.read_get(reader) self.assertEquals(MemcacheResult.OK, result) self.assertEquals(10, len(values)) #multi get pipeline, e.g. write N gets, but don't read out the results yet for i in range(0, N, 10): keys = ['test%d' % x for x in range(i, i + 10)] protocol.write_get(writer, keys) writer.flush() #now read the results for i in range(0, N, 10): result, values = protocol.read_get(reader) self.assertEquals(10, len(values)) self.assertTrue(('test%d' % i) in values) #pipelined multiget with same set of keys protocol.write_get(writer, [ 'test2', 'test8', 'test9', 'test11', 'test23', 'test24', 'test29', 'test31', 'test34' ]) writer.flush() protocol.write_get(writer, [ 'test2', 'test8', 'test9', 'test11', 'test23', 'test24', 'test29', 'test31', 'test34' ]) writer.flush() result1 = protocol.read_get(reader) result2 = protocol.read_get(reader) self.assertEquals(result1, result2)
def _send(self, identifier, body, timeout=30): try: conn = Socket.connect((self.host, self.port), timeout) try: req = u"POST / HTTP/1.1\r\nHost: %s\r\nContent-length: %d\r\nX-Realplexor: %s=%s%s\r\n\r\n%s" % ( self.host, len(body), (self.identifier or ""), ("%s:%s@" % (self.login, self.password) if self.login else ""), identifier or "", body) if type(req) == unicode: req = req.encode("utf-8") reader = BufferedReader(conn, Buffer(1024)) writer = BufferedWriter(conn, Buffer(1024)) writer.write_bytes(req) writer.flush() conn.socket.shutdown(socket.SHUT_WR) response = "" while True: try: chunk = reader.read_bytes_available() except EOFError: chunk = None if not chunk: break response += chunk if response: m = re_split_headers.split(response, 1) if not m: raise RealplexorError( "Non-HTTP response received:\n%s" % response) headers = m[0] body = m[1] m = re_http_status_line.match(headers) if not m: raise RealplexorError( "Non-HTTP response received:\n%s" % response) status, code = m.group(1, 2) if code != "200": raise RealplexorError("Request failed: %s\n%s" % (status, body)) m = re_content_length.search(headers) if not m: raise RealplexorError( "No Content-Length header in response headers:\n%s" % headers) expected_len = int(m.group(1)) if len(body) != expected_len: raise RealplexorError( "Response length (%d) is different from one specified in the Content-Length header (%d): possibly broken response" % (len(body), expected_len)) return body return response finally: conn.close() except socket.error as e: raise RealplexorSocketError("Socket error: %s" % e) except socket.gaierror as e: raise RealplexorError("DNS error: %s" % e) except socket.timeout as e: raise RealplexorError("Timeout error: %s" % e) except IOError as e: raise RealplexorSocketError("IOError: %s" % e)