class ToolsLib(object): log = None artifactPath = "install" def __init__(self): self.log = Logger() def findFile(self, needle, extension): # Find the zip file (We have to find like this because the date changes) found = False for file in os.listdir(self.artifactPath): if string.find(file, needle) >= 0 and re.search(extension + "$", file, re.IGNORECASE): found = True break; if not found: self.log.debug("files=%s" % ','.join(os.listdir(self.artifactPath))) raise Exception("Unable to find a zip file that has [%s] in the name with [%s] extension" % (needle, extension)) # Get the name of the file minus the extension reobj = re.compile(r"(?P<filename>.*?)(?P<ext>\.[0-9a-z]*$)", re.IGNORECASE) return reobj.search(file).group("filename")
def run(self, options): """ Runs the application. 'options' contains the CLI options dictionary. """ # Setup logging if options.debug: Logger.set_log_level(Logger.DEBUG) # Load configuration self.config = ConfigParser.RawConfigParser() self.config.read(options.config) MidiEngine.initialize() Logger.debug("Command line options: %s" % (options)) self.setup() if options.list_devices: self.list_devies() elif options.interactive: self.interactive_mode() else: self.loop()
def __set_state(self, state): self.__state = state Logger.debug("Switching to state %s" % (self.state_names[self.__state])) if self.__state == self.STATE_NORMAL: self.__update_levels(127, 0, 0) self.send_ctrl('master', 127) time.sleep(self.SWITCH_INTERVAL) self.send_ctrl('send1', 0) self.send_ctrl('send2', 0) elif self.__state == self.STATE_MUTE: self.__update_levels(0, 0, 0) self.send_ctrl('master', 0) self.send_ctrl('send1', 0) self.send_ctrl('send2', 0) elif self.__state == self.STATE_FX1: self.__update_levels(0, 127, 0) self.send_ctrl('send1', 127) time.sleep(self.SWITCH_INTERVAL) self.send_ctrl('master', 0) self.send_ctrl('send2', 0) elif self.__state == self.STATE_FX2: self.__update_levels(0, 0, 127) time.sleep(self.SWITCH_INTERVAL) self.send_ctrl('send2', 127) self.send_ctrl('master', 0) self.send_ctrl('send1', 0) self.send_ctrl('selector', (self.__state / 3.0) * 127.0)
class MarketProtocol(RPCProtocol): implements(MessageProcessor) def __init__(self, node_proto, router): self.router = router RPCProtocol.__init__(self, node_proto, router) self.log = Logger(system=self) self.handled_commands = [GET_CONTRACT, GET_IMAGE] self.multiplexer = None self.hashmap = HashMap() def connect_multiplexer(self, multiplexer): self.multiplexer = multiplexer def rpc_get_contract(self, sender, contract_hash): self.log.info("Looking up contract ID %s" % contract_hash.encode('hex')) self.router.addContact(sender) try: with open(self.hashmap.get_file(contract_hash), "r") as file: contract = file.read() return [contract] except: return ["None"] def rpc_get_image(self, sender, image_hash): self.log.info("Looking up image with hash %s" % image_hash.encode('hex')) self.router.addContact(sender) try: with open(self.hashmap.get_file(image_hash), "r") as file: image = file.read() return [image] except: return ["None"] def callGetContract(self, nodeToAsk, contract_hash): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_contract(address, contract_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetImage(self, nodeToAsk, image_hash): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_image(address, image_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def handleCallResponse(self, result, node): """ If we get a response, add the node to the routing table. If we get no response, make sure it's removed from the routing table. """ if result[0]: self.log.info("got response from %s, adding to router" % node) self.router.addContact(node) else: self.log.debug("no response from %s, removing from router" % node) self.router.removeContact(node) return result def __iter__(self): return iter(self.handled_commands)
def by_key(key): if RatingFactory.__lib.has_key(key): return RatingFactory.__lib[key] Logger.debug(u"Rating.by_key((%s, %s, %s, %s, %s, %s, %s, %s, %s)): create from key" % ( key[0], key[1], key[2], key[3], key[4], key[5], key[6], key[7], key[8])) return RatingFactory.get(key[0], key[1], key[2], key[3], key[4], key[5], key[6], key[7], key[8])
def __receive(self, input, msg): # Filter message if it does not belong to the configured MIDI channel if msg.getChannel() != self.channel: return Logger.debug("[%s] Received message: %s" % (self.name, MidiEngine.dump_msg(msg))) # Dispatch received message if self.receive: self.receive(self, msg)
def load(input): for i in range(len(Factories.__lib)): Logger.debug(u"Factories.load(): loading %s" % i) k = pickle.load(input) if k != i: # FIXME: we should die here Logger.error(u"Factories.load(): key mismatch %d vs %d" % (i, k)) Factories.__lib[k].setstate(pickle.load(input))
def __init__(self, name, input, output, channel): Logger.debug("[%s] Initializing channel: input='%s', output='%s, channel=%d" % (name, input, output, channel)) self.name = name self.input = MidiEngine.get_input_by_name(input) self.output = MidiEngine.get_output_by_name(output) self.channel = channel self.receive = None if self.input: self.input.receive = self.__receive
def __switch(self, fx): self.__state[fx] = not self.__state[fx] Logger.debug("Switching state of fx%d to %d" % (fx, self.__state[fx])) if self.__state[fx]: value = 127 else: value = 0 self.send_ctrl(['fx1', 'fx2', 'fx3', 'fx4'][fx], value) self.set_led(['led1', 'led2', 'led3', 'led4'][fx], self.__state[fx])
class Server(object): def __init__(self, kserver, signing_key, database): """ A high level class for sending direct, market messages to other nodes. A node will need one of these to participate in buying and selling. Should be initialized after the Kademlia server. """ self.kserver = kserver self.signing_key = signing_key self.router = kserver.protocol.router self.db = database self.log = Logger(system=self) self.protocol = MarketProtocol(kserver.node, self.router, signing_key, database) # TODO: we need a loop here that republishes keywords when they are about to expire # TODO: we also need a loop here to delete expiring contract (if they are set to expire) def querySeed(self, list_seed_pubkey): """ Query an HTTP seed for known vendors and save the vendors to the db. Args: Receives a list of one or more tuples Example [(seed, pubkey)] seed: A `string` consisting of "ip:port" or "hostname:port" pubkey: The hex encoded public key to verify the signature on the response """ for sp in list_seed_pubkey: seed, pubkey = sp try: self.log.debug("querying %s for vendors" % seed) c = httplib.HTTPConnection(seed) c.request("GET", "/?type=vendors") response = c.getresponse() self.log.debug("Http response from %s: %s, %s" % (seed, response.status, response.reason)) data = response.read() reread_data = data.decode("zlib") proto = peers.PeerSeeds() proto.ParseFromString(reread_data) verify_key = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.HexEncoder) verify_key.verify("".join(proto.serializedNode), proto.signature) v = self.db.VendorStore() for peer in proto.serializedNode: try: n = objects.Node() n.ParseFromString(peer) v.save_vendor(n.guid.encode("hex"), peer) except Exception: pass except Exception, e: self.log.error("failed to query seed: %s" % str(e))
class HeartbeatProtocol(protocol.Protocol): """ For listening on the libbitcoin server heartbeat port """ def __init__(self, libbitcoin_client): self.libbitcoin_client = libbitcoin_client self.timeout = reactor.callLater(7, self.call_timeout) self.log = Logger(system=self) def call_timeout(self): self.log.critical("Libbitcoin server offline") self.libbitcoin_client.connected = False def dataReceived(self, data): self.log.debug("libbitcoin heartbeat") self.timeout.cancel() self.libbitcoin_client.connected = True self.transport.loseConnection()
class MarketProtocol(RPCProtocol): implements(MessageProcessor) def __init__(self, node_proto, router): self.router = router RPCProtocol.__init__(self, node_proto, router) self.log = Logger(system=self) self.handled_commands = [GET_CONTRACT] def connect_multiplexer(self, multiplexer): self.multiplexer = multiplexer def rpc_get_contract(self, sender, contract_hash): self.log.info("Looking up contract ID" % long(contract_hash.encode('hex'), 16)) self.router.addContact(sender) try: with open(get_data_folder() + "/Store/Listings/Contracts/" + str(long(contract_hash.encode('hex'), 16)) + '.json') as contract_file: data = json.load(contract_file) return str(data) except: return "None" def call_get_contract(self, nodeToAsk, contract_hash): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_contract(address, contract_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def handleCallResponse(self, result, node): """ If we get a response, add the node to the routing table. If we get no response, make sure it's removed from the routing table. """ if result[0]: self.log.info("got response from %s, adding to router" % node) self.router.addContact(node) else: self.log.debug("no response from %s, removing from router" % node) self.router.removeContact(node) return result def __iter__(self): return iter(self.handled_commands)
class Crawler(threading.Thread): def __init__(self, cookie, manager): threading.Thread.__init__(self) cookie_handler = urllib2.HTTPCookieProcessor(cookie) self._opener = urllib2.build_opener(cookie_handler) self._manager = manager self._logger = Logger(u"crawler.log") def run(self): while True: try: # fetch url url = self._manager.get_url() self._logger.debug("Fetch a url " + url) # fetch page request = urllib2.Request(url) page = self._opener.open(request) self._logger.debug("Fetch the page for url " + url) # insert page while True: try: self._manager.insert_page(url, page) self._logger.debug("Insert the page") break except PageQueueFullForNowError: time.sleep(0.1) except urllib2.URLError: self._manager.insert_page(url, None) self._logger.error("URLError for " + url) except UrlQueueEmptyForNowError: time.sleep(0.1) except NoUrlToCrawlError: break
def validate_letter_target(params): try: target = params['target'] if target not in ['public', 'friend', 'subscriber', 'me', 'users', 'email']: raise MultipleInvalid('hello') elif target == 'public': assert isinstance(params['category_id'], int) assert params['category_id'] > 0 elif target == 'users': assert isinstance(params['user_ids'], list) assert len(params['user_ids']) > 0 for id in params['user_ids']: assert isinstance(id, int) elif target == 'email': Logger.debug('x') assert isinstance(params['email'], unicode) or isinstance(params['email'], str) return True except Exception as e: Logger.debug('y') raise MultipleInvalid(str(e))
def handle(self): while True: self.data = self.request.recv(1024).strip() # cur_thread = threading.currentThread() info = "RECV from ", self.client_address[0], self.client_address[1] Logger.info(info) if self.data == None or len(self.data) == 0: Logger.error("empty command") break Logger.debug(self.data) # business logic here try: cmd = command.decode(self.data) except: Logger.error("format error") break # try: response = application.dispatch(cmd) Logger.debug(response) self.send(response) if response["code"] == -1: # server.shutdown() break
class SpiderCrawl(object): """ Crawl the network and look for given 160-bit keys. """ def __init__(self, protocol, node, peers, ksize, alpha): """ Create a new C{SpiderCrawl}er. Args: protocol: A :class:`~kademlia.protocol.KademliaProtocol` instance. node: A :class:`~kademlia.node.Node` representing the key we're looking for peers: A list of :class:`~kademlia.node.Node` instances that provide the entry point for the network ksize: The value for k based on the paper alpha: The value for alpha based on the paper """ self.protocol = protocol self.ksize = ksize self.alpha = alpha self.node = node self.nearest = NodeHeap(self.node, self.ksize) self.lastIDsCrawled = [] self.log = Logger(system=self) self.log.debug("creating spider with peers: %s" % peers) self.nearest.push(peers) def _find(self, rpcmethod): """ Get either a value or list of nodes. Args: rpcmethod: The protocol's callFindValue or callFindNode. The process: 1. calls find_* to current ALPHA nearest not already queried nodes, adding results to current nearest list of k nodes. 2. current nearest list needs to keep track of who has been queried already sort by nearest, keep KSIZE 3. if list is same as last time, next call should be to everyone not yet queried 4. repeat, unless nearest list has all been queried, then ur done """ self.log.debug("crawling with nearest: %s" % str(tuple(self.nearest))) count = self.alpha if self.nearest.getIDs() == self.lastIDsCrawled: self.log.debug("last iteration same as current - checking all in list now") count = len(self.nearest) self.lastIDsCrawled = self.nearest.getIDs() ds = {} for peer in self.nearest.getUncontacted()[:count]: ds[peer.id] = rpcmethod(peer, self.node) self.nearest.markContacted(peer) return deferredDict(ds).addCallback(self._nodesFound)
class User(object): def __init__(self): self.httpRequest = HttpRequest() self.httpParser = HttpParser() self.redisConn = RedisConnect() self.logger = Logger() self.user_file = open("/opt_c/dianping/file/user_urls_4.txt", "a") #self.mysqlConn = MysqlClient("127.0.0.1","root","homelink",'dianping',3306) self.mysqlConn = MysqlPool() def saveHtml(self, url, param, html, page): id = re.findall('[0-9]+', url)[0] print(id) # path = '/Users/homelink/dianping/html/'+param+'/'+id[0:3]+'/'+id[3:6]+'/' path = '/opt_c/dianping/html/' + param + '/' + id[0:3] + '/' + id[ 3:6] + '/' if os.path.exists(path) == False: os.makedirs(path) html_path = path + id + '_' + param + '_' + str(page) + '.txt' f = open(html_path, "a") f.write(html) f.flush() def run(self): while self.redisConn.scard("dianping::store") > 0: # while self.redisConn.scard("test")>0: store = self.redisConn.pop("dianping::store") # store = self.redisConn.pop("test") url = "http://www.dianping.com" + store + '/review_more' print(url) self.logger.info(url) dic_list = [ "user_url", "user_name", "user_image", "user_level", "create_time", "update_time" ] postDic = {} page = 0 count = 20 while count == 20: page = page + 1 count = 0 try: print(url + '?pageno=' + str(page)) html, code = self.httpRequest.get(url + '?pageno=' + str(page)) print(code) if code == 404 or code == 403 or code == 429: print("match error stop !!!") self.redisConn.sadd("failed::store::user_1", url + '?pageno=' + str(page)) self.redisConn.sadd("failed::store", store) time.sleep(60 * 20) else: # time.sleep(random.randint(3,6)) time.sleep(2) self.saveHtml(store, "user", html, page) print(url + '?pageno=' + str(page)) self.logger.info(url + '?pageno=' + str(page)) sites = self.httpParser.parseNode( html, '//div[@class="comment-list"]/ul/li') print(sites[0]) for site in sites: user_url = site.xpath('div/a/@href') print(user_url[0]) self.redisConn.sadd("dianping::review::user", *user_url) self.redisConn.sadd("dianping::wish::user", *user_url) self.redisConn.sadd("dianping::checkin::user", *user_url) user_name = site.xpath('div/p/a/text()') user_image = site.xpath('div/a/img/@src') user_level = site.xpath('div/p[2]/span/@class') postDic["user_url"] = user_url[0] postDic["user_name"] = user_name[0].replace( "'", "") postDic["user_image"] = user_image[0] postDic["store"] = store if user_level: postDic["user_level"] = user_level[0] else: postDic["user_level"] = '' postDic["create_time"] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) postDic["update_time"] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) count = len(sites) # self.mysqlConn.insert(dic_list,"user",**postDic) line = json.dumps(dict(postDic), ensure_ascii=False) self.user_file.write(line + '\n') self.user_file.flush() self.redisConn.sadd("success::store::user", url + '?pageno=' + str(page)) self.redisConn.sadd("success::store", store) except: # self.redisConn.sadd("failed::store::user",url+'?pageno='+str(page)) print(sys.exc_info()) self.redisConn.sadd("failed::store::user_1", url + '?pageno=' + str(page)) self.redisConn.sadd("failed::store", store) self.logger.debug("start UserUrl:" + url + ' error :' + str(sys.exc_info()[0]) + ',' + str(sys.exc_info()[1]) + ',' + str(sys.exc_info()[2])) time.sleep(10) def UserReviewTrade(self): while self.redisConn.scard("dianping::review::user") > 0: # while self.redisConn.scard("test")>0: user = self.redisConn.pop("dianping::review::user") # user = self.redisConn.pop("test") url = "http://www.dianping.com" + user print(url) postDic = {} dic_list = [ "user_url", "user_name", "store_name", "store_url", "store_score", "store_location", "review_time", "crawl_time", "create_time", "update_time" ] count = 15 page = 1 while count == 15: try: review_html = self.httpRequest.get(url + '/reviews' + '?pg=' + str(page) + '&reviewCityId=2') print(url + '/reviews' + '?pg=' + str(page) + '&reviewCityId=2') sites = self.httpParser.parseHref( review_html, '//div[@id="J_review"]/div[@class="pic-txt"]/ul/li') for site in sites: store_url = site.xpath('div/div[1]/h6/a/@href') print(store_url) store_name = site.xpath('div/div[1]/h6/a/text()') print(store_name[0]) store_score = site.xpath( 'div/div[2]/div[2]/span/@class') store_location = site.xpath( 'div/div[2]/div[1]/p/text()') review_time = site.xpath( 'div/div[2]/div[@class="mode-tc info"]/span[1]/text()' ) review_crawl_time = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) postDic["user_url"] = user postDic["user_name"] = '' postDic["store_name"] = store_name[0] postDic["store_url"] = store_url[0] postDic["store_score"] = store_score[0] postDic["store_location"] = store_location[0] postDic["review_time"] = review_time[0] postDic["crawl_time"] = review_crawl_time[0] postDic["create_time"] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) postDic["update_time"] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) #self.mysqlConn.insert(dic_list,"user_review",**postDic) self.redisConn.sadd("success::review", user) page = page + 1 count = len(sites) except: print(sys.exc_info()) self.redisConn.sadd("failed::review", user) def UserWishTrade(self): # while self.redisConn.scard("dianping::wish::user")>0: while self.redisConn.scard("test") > 0: # user = self.redisConn.pop("dianping::wish::user") user = self.redisConn.pop("test") url = "http://www.dianping.com" + user dic_list = [ "user_url", "user_name", "store_name", "store_url", "store_score", "store_location", "wish_time", "crawl_time", "create_time", "update_time" ] postDic = {} count = 30 page = 1 while count == 30: try: wish_html = self.httpRequest.get(url + '/wishlists?pg=' + str(page) + '&favorTag=s-1_c-1_t-1') sites = self.httpParser.parseNode( wish_html, '//div[@class="pic-txt favor-list"]/ul/li') for site in sites: wish_store_url = site.xpath('div/div[1]/h6/a/@href') wish_store_name = site.xpath('div/div[1]/h6/a/text()') wish_store_score = site.xpath( 'div/div[2]/div/p/span[2]/@class') wish_store_location = site.xpath( 'div/div[2]/div[1]/p/text()') wish_time = site.xpath( 'div/div[2]/div[2]/span/i/text()') wish_crawl_time = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) postDic["user_url"] = user postDic["user_name"] = '' postDic["store_name"] = wish_store_name[0] postDic["store_url"] = wish_store_url[0] postDic["store_score"] = wish_store_score[0] postDic["store_location"] = wish_store_location[0] postDic["wish_time"] = wish_time[0] postDic["crawl_time"] = wish_crawl_time[0] postDic["create_time"] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) postDic["update_time"] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) #self.mysqlConn.insert(dic_list,"user_wish",**postDic) self.redisConn.sadd("success::wish", user) page = page + 1 count = len(sites) except: self.redisConn.sadd("failed::wish", user) print(sys.exc_info()) def UserCheckinTrade(self): # while self.redisConn.scard("dianping::checkin::user")>0: while self.redisConn.scard("test") > 0: # member = self.redisConn.pop("dianping::checkin::user") member = self.redisConn.pop("test") url = "http://www.dianping.com" + member memberId = member.split("/")[2] dic_list = [ "user_url", "user_name", "store_name", "store_url", "store_location", "checkin_time", "crawl_time", "create_time", "update_time" ] postDic = {} try: checkin_html = self.httpRequest.get(url + '/checkin') print(url + '/checkin') total_count = self.httpParser.parseText( checkin_html, '//div[@class="pic-txt head-user"]/div[2]/div[3]/ul/li[4]/a/text()' ) total = re.findall("[1-9]+", total_count[0].encode("utf-8"))[0] page = int(total) / 20 sites = self.httpParser.parseNode(checkin_html, '//ul[@id="J_list"]/li') for site in sites: checkin_store_url = site.xpath('h6/a/@href') checkin_store_name = site.xpath('h6/a/text()') checkin_store_location = site.xpath('p/text()') checkin_time = site.xpath('h6/span/text()') chechin_crawl_time = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) postDic["user_url"] = member postDic["user_name"] = '' postDic["store_name"] = checkin_store_name[0].replace( "'", "") print(checkin_store_name[0]) postDic["store_url"] = checkin_store_url[0] postDic["store_location"] = checkin_store_location[0] postDic["checkin_time"] = checkin_time[0] postDic["crawl_time"] = chechin_crawl_time[0] postDic["create_time"] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) postDic["update_time"] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) #self.mysqlConn.insert(dic_list,"user_checkin",**postDic) if page: for i in range(page): time.sleep(5) data_form = { "memberId": str(memberId), "page": str(i + 2) } url = "http://www.dianping.com/ajax/member/checkin/checkinList" result = self.httpRequest.post(url, data_form) result_list = json.loads(result)["msg"]["checkinList"] for checkin in result_list: postDic["user_url"] = member postDic["user_name"] = '' postDic["store_name"] = checkin[ "shopName"].replace("'", "") print(checkin["shopName"]) postDic["store_url"] = '' postDic["store_location"] = checkin["shopAddress"] postDic["checkin_time"] = checkin["time"] postDic["crawl_time"] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) postDic["create_time"] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) postDic["update_time"] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) #self.mysqlConn.insert(dic_list,"user_checkin",**postDic) self.redisConn.sadd("success::checkin", member) except: self.redisConn.sadd("failed::checkin", member) print(sys.exc_info())
def send(self, msg): # Override MIDI channel msg.setChannel(self.channel) Logger.debug("[%s] Sending message: %s" % (self.name, MidiEngine.dump_msg(msg))) if self.output: self.output.send(msg)
class UrlRunnable: def __init__(self): self.httpRequest = HttpRequest() self.httpParser = HttpParser() self.redisConn = RedisConnect() self.start_url = "http://www.dianping.com/shopall/2/0" self.logger = Logger() # self.mysqlConn = MysqlClient("127.0.0.1","root","homelink",'dianping',3306) self.mysqlConn = MysqlPool() self.store_file = open("/opt_c/dianping/file/store_urls.txt", "a") # self.store_file = open("/homelink/dianping/file/store_urls.txt","a") def saveHtml(self, url, param, html): id = re.findall('[0-9]+', url)[0] # path = '/Users/homelink/dianping/html/'+param+'/'+id[0:3]+'/'+id[3:6]+'/' path = '/opt_c/dianping/html/' + param + '/' + id[0:3] + '/' + id[ 3:6] + '/' if os.path.exists(path) == False: os.makedirs(path) html_path = path + id + '_' + param + '.txt' f = open(html_path, "a") f.write(html) f.flush() def regUrl(self, link): try: url = "http://www.dianping.com" + link print(url) html, code = self.httpRequest.get(url) time.sleep(2) # hrefs = self.httpParser.parseHref(html,'//div[@id="region-nav"]/a/@href') hrefs = self.httpParser.parseHref( html, '//div[@id="region-nav-sub"]/a/@href') print(hrefs) self.redisConn.sadd("dianping::tag::reg_sub", *hrefs) except: print(sys.exc_info()) self.redisConn.sadd("failed::tag::reg_sub", link) def linksUrl(self): try: html, code = self.httpRequest.get(self.start_url) print(code) sites = self.httpParser.parseNode( html, '//div[@class="main_w"]/div/div[1]/dl[17]') print(sites) postDic = {} dic_list = [ "tag_level_1", "tag_level_2", "tag_link", "create_time", "update_time" ] for site in sites: tags = site.xpath('dt/a/text()') link_urls = site.xpath('dd/ul/li/a/@href') print(link_urls) # self.redisConn.sadd("dianping::tag",*link_urls) link_tags = site.xpath('dd/ul/li/a/text()') for i in range(len(link_tags)): postDic["tag_level_1"] = tags[0] postDic["tag_level_2"] = link_tags[i] postDic["tag_link"] = link_urls[i] self.regUrl(link_urls[i]) postDic["create_time"] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) postDic["update_time"] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) # self.mysqlConn.insert(dic_list,"storeTag",**postDic) except: print(sys.exc_info()) #return link_url def run(self): # while self.redisConn.scard("dianping::tag")>0: while self.redisConn.scard("dianping::tag::reg") > 0: # while self.redisConn.scard("test_1")>0: # tag = self.redisConn.pop("dianping::tag") tag = self.redisConn.pop("dianping::tag::reg") # tag = self.redisConn.pop("test_1") url = "http://www.dianping.com" + tag print(url) self.logger.info("start StoreUrl:" + url) postDic = {} dic_list = [ "store_url", "store_name", "father_url", "father_tag", "store_score", "trade_area", "location", "cost", "review", "create_time", "update_time", "longitude", "latitude" ] page = 0 count = 19 while count >= 15 and page <= 50: page = page + 1 count = 0 try: html, code = self.httpRequest.get(url + 'p' + str(page)) time.sleep(2) print(code) self.logger.info("start StoreUrl: " + url + 'p' + str(page)) print("start StoreUrl: " + url + 'p' + str(page)) sites = self.httpParser.parseNode( html, '//div[@id="shop-all-list"]/ul/li') print(sites) count = len(sites) for site in sites: store_urls = site.xpath('div[2]/div[1]/a[1]/@href') time.sleep(2) store_html, code = self.httpRequest.get( "http://www.dianping.com" + store_urls[0]) print("http://www.dianping.com" + store_urls[0]) self.logger.info( "storeUrl request : http://www.dianping.com" + store_urls[0]) extract_address = re.findall("({lng:(.*),lat:(.*)})", store_html) if extract_address: longitude = extract_address[0][1] latitude = extract_address[0][2] postDic["longitude"] = longitude postDic["latitude"] = latitude self.saveHtml(store_urls[0], "store", store_html) store_names = site.xpath('div[2]/div[1]/a[1]/@title') father_tag = site.xpath( 'div[2]/div[3]/a[1]/span/text()') store_score = site.xpath('div[2]/div[2]/span/@class') trade_area = site.xpath( 'div[2]/div[3]/a[2]/span/text()') location = site.xpath( 'div[2]/div[3]/span[@class="addr"]/text()') cost = site.xpath('div[2]/div[2]/a[2]/b/text()') review = site.xpath('div[2]/div[2]/a/b/text()') father_url = tag if not self.redisConn.sismember( "dianping::store::bak", store_urls[0]): self.redisConn.sadd("dianping::store", store_urls[0]) self.redisConn.sadd("dianping::store::bak", store_urls[0]) postDic["store_url"] = store_urls[0] postDic["store_name"] = store_names[0].replace("'", "") self.logger.info(store_names[0]) postDic["father_url"] = tag postDic["father_tag"] = father_tag[0] postDic["store_score"] = store_score[0] if trade_area: postDic["trade_area"] = trade_area[0] else: postDic["trade_area"] = '' postDic["location"] = location[0] if cost: postDic["cost"] = cost[0] else: postDic["cost"] = '' if review: postDic["review"] = review[0] else: postDic["review"] = '' postDic["create_time"] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) postDic["update_time"] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) line = json.dumps(dict(postDic), ensure_ascii=False) self.store_file.write(line + '\n') self.store_file.flush() self.mysqlConn.insert(dic_list, "store", **postDic) self.redisConn.sadd("success::tag::url", url + 'p' + str(page)) except: self.redisConn.sadd("failed::tag::url", url + 'p' + str(page)) self.redisConn.sadd("failed::tag", tag) self.logger.debug("start StoreUrl:" + url + 'p' + str(page) + ' error :' + str(sys.exc_info()[0]) + ',' + str(sys.exc_info()[1]) + ',' + str(sys.exc_info()[2])) print(sys.exc_info())
class KademliaProtocol(RPCProtocol): implements(MessageProcessor) def __init__(self, sourceNode, storage, ksize, database): self.ksize = ksize self.router = RoutingTable(self, ksize, sourceNode) self.storage = storage self.sourceNode = sourceNode self.multiplexer = None self.db = database self.log = Logger(system=self) self.handled_commands = [ PING, STUN, STORE, DELETE, FIND_NODE, FIND_VALUE, HOLE_PUNCH, INV, VALUES ] RPCProtocol.__init__(self, sourceNode, self.router) def connect_multiplexer(self, multiplexer): self.multiplexer = multiplexer def getRefreshIDs(self): """ Get ids to search for to keep old buckets up to date. """ ids = [] for bucket in self.router.getLonelyBuckets(): ids.append(random.randint(*bucket.range)) return ids def rpc_stun(self, sender): self.addToRouter(sender) return [sender.ip, str(sender.port)] def rpc_ping(self, sender): self.addToRouter(sender) return [self.sourceNode.getProto().SerializeToString()] def rpc_store(self, sender, keyword, key, value, ttl): self.addToRouter(sender) self.log.debug("got a store request from %s, storing value" % str(sender)) if len(keyword) == 20 and len(key) <= 33 and len( value) <= 2100 and int(ttl) <= 604800: self.storage[keyword] = (key, value, int(ttl)) return ["True"] else: return ["False"] def rpc_delete(self, sender, keyword, key, signature): self.addToRouter(sender) value = self.storage.getSpecific(keyword, key) if value is not None: # Try to delete a message from the dht if keyword == digest(sender.id): try: verify_key = nacl.signing.VerifyKey( sender.signed_pubkey[64:]) verify_key.verify(key, signature) self.storage.delete(keyword, key) return ["True"] except Exception: return ["False"] # Or try to delete a pointer else: try: node = objects.Node() node.ParseFromString(value) pubkey = node.signedPublicKey[64:] try: verify_key = nacl.signing.VerifyKey(pubkey) verify_key.verify(signature + key) self.storage.delete(keyword, key) return ["True"] except Exception: return ["False"] except Exception: pass return ["False"] def rpc_find_node(self, sender, key): self.log.debug("finding neighbors of %s in local table" % key.encode('hex')) self.addToRouter(sender) node = Node(key) nodeList = self.router.findNeighbors(node, exclude=sender) ret = [] for n in nodeList: ret.append(n.getProto().SerializeToString()) return ret def rpc_find_value(self, sender, keyword): self.addToRouter(sender) ret = ["value"] value = self.storage.get(keyword, None) if value is None: return self.rpc_find_node(sender, keyword) ret.extend(value) return ret def rpc_inv(self, sender, *serlialized_invs): self.addToRouter(sender) ret = [] for inv in serlialized_invs: try: i = objects.Inv() i.ParseFromString(inv) if self.storage.getSpecific(i.keyword, i.valueKey) is None: ret.append(inv) except Exception: pass return ret def rpc_values(self, sender, *serialized_values): self.addToRouter(sender) for val in serialized_values: try: v = objects.Value() v.ParseFromString(val) self.storage[v.keyword] = (v.valueKey, v.serializedData, int(v.ttl)) except Exception: pass return ["True"] def callFindNode(self, nodeToAsk, nodeToFind): address = (nodeToAsk.ip, nodeToAsk.port) d = self.find_node(address, nodeToFind.id) return d.addCallback(self.handleCallResponse, nodeToAsk) def callFindValue(self, nodeToAsk, nodeToFind): address = (nodeToAsk.ip, nodeToAsk.port) d = self.find_value(address, nodeToFind.id) return d.addCallback(self.handleCallResponse, nodeToAsk) def callPing(self, nodeToAsk): address = (nodeToAsk.ip, nodeToAsk.port) d = self.ping(address) return d.addCallback(self.handleCallResponse, nodeToAsk) def callStore(self, nodeToAsk, keyword, key, value, ttl): address = (nodeToAsk.ip, nodeToAsk.port) d = self.store(address, keyword, key, value, str(int(round(ttl)))) return d.addCallback(self.handleCallResponse, nodeToAsk) def callDelete(self, nodeToAsk, keyword, key, signature): address = (nodeToAsk.ip, nodeToAsk.port) d = self.delete(address, keyword, key, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callInv(self, nodeToAsk, serlialized_inv_list): address = (nodeToAsk.ip, nodeToAsk.port) d = self.inv(address, *serlialized_inv_list) return d.addCallback(self.handleCallResponse, nodeToAsk) def callValues(self, nodeToAsk, serlialized_values_list): address = (nodeToAsk.ip, nodeToAsk.port) d = self.values(address, *serlialized_values_list) return d.addCallback(self.handleCallResponse, nodeToAsk) def transferKeyValues(self, node): """ Given a new node, send it all the keys/values it should be storing. @param node: A new node that just joined (or that we just found out about). Process: For each key in storage, get k closest nodes. If newnode is closer than the furtherst in that list, and the node for this server is closer than the closest in that list, then store the key/value on the new node (per section 2.5 of the paper) """ def send_values(inv_list): values = [] if inv_list[0]: for requested_inv in inv_list[1]: try: i = objects.Inv() i.ParseFromString(requested_inv) value = self.storage.getSpecific(i.keyword, i.valueKey) if value is not None: v = objects.Value() v.keyword = i.keyword v.valueKey = i.valueKey v.serializedData = value v.ttl = int( round( self.storage.get_ttl( i.keyword, i.valueKey))) values.append(v.SerializeToString()) except Exception: pass if len(values) > 0: self.callValues(node, values) inv = [] for keyword in self.storage.iterkeys(): keynode = Node(keyword) neighbors = self.router.findNeighbors(keynode, exclude=node) if len(neighbors) > 0: newNodeClose = node.distanceTo( keynode) < neighbors[-1].distanceTo(keynode) thisNodeClosest = self.sourceNode.distanceTo( keynode) < neighbors[0].distanceTo(keynode) if len(neighbors) == 0 \ or (newNodeClose and thisNodeClosest) \ or (thisNodeClosest and len(neighbors) < self.ksize): # pylint: disable=W0612 for k, v in self.storage.iteritems(keyword): i = objects.Inv() i.keyword = keyword i.valueKey = k inv.append(i.SerializeToString()) if len(inv) > 0: self.callInv(node, inv).addCallback(send_values) def handleCallResponse(self, result, node): """ If we get a response, add the node to the routing table. If we get no response, make sure it's removed from the routing table. """ if result[0]: if self.router.isNewNode(node): self.transferKeyValues(node) self.router.addContact(node) else: self.log.debug("no response from %s, removing from router" % node) self.router.removeContact(node) return result def addToRouter(self, node): """ Called by rpc_ functions when a node sends them a request. We add the node to our router and transfer our stored values if they are new and within our neighborhood. """ if self.router.isNewNode(node): self.log.debug("Found a new node, transferring key/values") reactor.callLater(1, self.transferKeyValues, node) self.router.addContact(node) def __iter__(self): return iter(self.handled_commands)
class QRCodeConverter: def __init__(self): self.util = Util() self.log = Logger() self.block_size = 8 self.circle_radius = self.block_size * 5 self.qrcode_img_name_prefix = "qrcode" self.qrcode_img_name_surffix = ".png" def set_inner_img_path(self, path): self.inner_img_path = path def set_qrcode_content(self, content): self.content = content def set_qrcode_size(self, size): self.size = size def get_qrcode_thumbnail_size(self): assert self.size assert isinstance(self.size, tuple) assert len(self.size) == 2 assert self.size[0] assert self.size[1] res_val = max(self.size[0], self.size[1]) return (res_val, res_val) def fetch_file(self, path): import os if path.startswith('http'): f = cStringIO.StringIO(urllib.urlopen(path).read()) else: assert os.path.isfile(path) f = cStringIO.StringIO(path) return f def process(self, successCallback, failureCallback): im = qrcode.make(self.content).convert('RGBA') outputname = self.qrcode_img_name_prefix + str( self.util.now()) + self.qrcode_img_name_surffix try: inner_img_file = self.fetch_file(self.inner_img_path) inner_img = Image.open(inner_img_file.read()).convert('RGBA') (width, height) = inner_img.size dim = height if (width > dim): dim = width xTrans = (im.size[0] - width) / 2.0 yTrans = (im.size[1] - dim) / 2.0 im.paste(inner_img, (int(xTrans), int(yTrans)), inner_img) try: (img_width, img_height) = get_qrcode_thumbnail_size() self.log.debug('Get width and height for qrcode image') self.log.debug('Set thunbnail for qrcode with size: (%d, %d)' % (img_width, img_height)) im.thumbnail(self.size) except Exception as thumbnail_exception: self.log.debug( 'Cannot get suitable width and height for qrcode image, do nothing' ) except Exception as e: self.log.warn('Cannot fetch inner image with path: %s' % (self.inner_img_path)) self.log.warn(e) im.save(outputname) self.log.info( "success to process, now call successCallback, qrcode file: " + outputname) successCallback(self)
class KademliaProtocol(RPCProtocol): implements(MessageProcessor) def __init__(self, sourceNode, storage, ksize, database): self.ksize = ksize self.router = RoutingTable(self, ksize, sourceNode) self.storage = storage self.sourceNode = sourceNode self.multiplexer = None self.db = database self.log = Logger(system=self) self.handled_commands = [PING, STUN, STORE, DELETE, FIND_NODE, FIND_VALUE, HOLE_PUNCH] RPCProtocol.__init__(self, sourceNode.getProto(), self.router) def connect_multiplexer(self, multiplexer): self.multiplexer = multiplexer def getRefreshIDs(self): """ Get ids to search for to keep old buckets up to date. """ ids = [] for bucket in self.router.getLonelyBuckets(): ids.append(random.randint(*bucket.range)) return ids def rpc_stun(self, sender): self.addToRouter(sender) return [sender.ip, str(sender.port)] def rpc_ping(self, sender): self.addToRouter(sender) return [self.sourceNode.getProto().SerializeToString()] def rpc_store(self, sender, keyword, key, value): self.addToRouter(sender) self.log.debug("got a store request from %s, storing value" % str(sender)) if len(keyword) == 20 and len(key) <= 33 and len(value) <= 1800: self.storage[keyword] = (key, value) return ["True"] else: return ["False"] def rpc_delete(self, sender, keyword, key, signature): self.addToRouter(sender) value = self.storage.getSpecific(keyword, key) if value is not None: # Try to delete a message from the dht if keyword == digest(sender.id): try: verify_key = nacl.signing.VerifyKey(sender.signed_pubkey[64:]) verify_key.verify(key, signature) self.storage.delete(keyword, key) return ["True"] except Exception: return ["False"] # Or try to delete a pointer else: try: node = objects.Node() node.ParseFromString(value) pubkey = node.signedPublicKey[64:] try: verify_key = nacl.signing.VerifyKey(pubkey) verify_key.verify(signature + key) self.storage.delete(keyword, key) return ["True"] except Exception: return ["False"] except Exception: pass return ["False"] def rpc_find_node(self, sender, key): self.log.info("finding neighbors of %s in local table" % key.encode('hex')) self.addToRouter(sender) node = Node(key) nodeList = self.router.findNeighbors(node, exclude=sender) ret = [] for n in nodeList: ret.append(n.getProto().SerializeToString()) return ret def rpc_find_value(self, sender, key): self.addToRouter(sender) ret = ["value"] value = self.storage.get(key, None) if value is None: return self.rpc_find_node(sender, key) ret.extend(value) return ret def callFindNode(self, nodeToAsk, nodeToFind): address = (nodeToAsk.ip, nodeToAsk.port) d = self.find_node(address, nodeToFind.id) return d.addCallback(self.handleCallResponse, nodeToAsk) def callFindValue(self, nodeToAsk, nodeToFind): address = (nodeToAsk.ip, nodeToAsk.port) d = self.find_value(address, nodeToFind.id) return d.addCallback(self.handleCallResponse, nodeToAsk) def callPing(self, nodeToAsk): address = (nodeToAsk.ip, nodeToAsk.port) d = self.ping(address) return d.addCallback(self.handleCallResponse, nodeToAsk) def callStore(self, nodeToAsk, keyword, key, value): address = (nodeToAsk.ip, nodeToAsk.port) d = self.store(address, keyword, key, value) return d.addCallback(self.handleCallResponse, nodeToAsk) def callDelete(self, nodeToAsk, keyword, key, signature): address = (nodeToAsk.ip, nodeToAsk.port) d = self.delete(address, keyword, key, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def transferKeyValues(self, node): """ Given a new node, send it all the keys/values it should be storing. @param node: A new node that just joined (or that we just found out about). Process: For each key in storage, get k closest nodes. If newnode is closer than the furtherst in that list, and the node for this server is closer than the closest in that list, then store the key/value on the new node (per section 2.5 of the paper) """ ds = [] for keyword in self.storage.iterkeys(): keynode = Node(keyword) neighbors = self.router.findNeighbors(keynode, exclude=node) if len(neighbors) > 0: newNodeClose = node.distanceTo(keynode) < neighbors[-1].distanceTo(keynode) thisNodeClosest = self.sourceNode.distanceTo(keynode) < neighbors[0].distanceTo(keynode) if len(neighbors) == 0 \ or (newNodeClose and thisNodeClosest) \ or (thisNodeClosest and len(neighbors) < self.ksize): for k, v in self.storage.iteritems(keyword): ds.append(self.callStore(node, keyword, k, v)) return defer.gatherResults(ds) def handleCallResponse(self, result, node): """ If we get a response, add the node to the routing table. If we get no response, make sure it's removed from the routing table. """ if result[0]: if self.router.isNewNode(node): self.transferKeyValues(node) self.log.info("got response from %s, adding to router" % node) self.router.addContact(node) else: self.log.debug("no response from %s, removing from router" % node) self.router.removeContact(node) return result def addToRouter(self, node): """ Called by rpc_ functions when a node sends them a request. We add the node to our router and transfer our stored values if they are new and within our neighborhood. """ if self.router.isNewNode(node): self.log.debug("Found a new node, transferring key/values") self.transferKeyValues(node) self.router.addContact(node) def __iter__(self): return iter(self.handled_commands)
class SelfSignedJwt(object): NumCharIn128BitHexString = 128/8*2 numCharIn160BitHexString = 160/8*2 ThumbprintRegEx = "^[a-f\d]*$" def __init__(self, call_context, authority, client_id): self._log = Logger('SelfSignedJwt', call_context['log_context']) self._call_context = call_context self._authortiy = authority self._token_endpoint = authority.token_endpoint self._client_id = client_id def _get_date_now(self): return datetime.datetime.now() def _get_new_jwt_id(self): return str(uuid.uuid4()) def _create_x5t_value(self, thumbprint): hex_str = thumbprint.replace(':', '').replace(' ', '') b64_str = base64.urlsafe_b64encode(hex_str.encode()) return b64_str.decode() def _create_header(self, thumbprint): x5t = self._create_x5t_value(thumbprint) header = {'typ':'JWT', 'alg':'RS256', 'x5t':x5t} self._log.debug("Creating self signed JWT header. x5t: {0}".format(x5t)) return header def _create_payload(self): now = self._get_date_now() minutes = datetime.timedelta(0, 0, 0, 0, Jwt.SELF_SIGNED_JWT_LIFETIME) expires = now + minutes self._log.debug('Creating self signed JWT payload. Expires: {0} NotBefore: {1}'.format(expires, now)) jwt_payload = {} jwt_payload[Jwt.AUDIENCE] = self._token_endpoint jwt_payload[Jwt.ISSUER] = self._client_id jwt_payload[Jwt.SUBJECT] = self._client_id jwt_payload[Jwt.NOT_BEFORE] = int(time.mktime(now.timetuple())) jwt_payload[Jwt.EXPIRES_ON] = int(time.mktime(expires.timetuple())) jwt_payload[Jwt.JWT_ID] = self._get_new_jwt_id() return jwt_payload def _raise_on_invalid_jwt_signature(self, encoded_jwt): segments = encoded_jwt.split('.') if len(segments) < 3 or not segments[2]: raise self._log.create_error('Failed to sign JWT. This is most likely due to an invalid certificate.') def _raise_on_invalid_thumbprint(self, thumbprint): thumbprint_sizes = [self.NumCharIn128BitHexString, self.numCharIn160BitHexString] if len(thumbprint) not in thumbprint_sizes or not re.search(self.ThumbprintRegEx, thumbprint): raise self._log.create_error("The thumbprint does not match a known format") def _sign_jwt(self, header, payload, certificate): # TODO: Might want to load the cert and get the string proper. cert_start_str = '-----BEGIN RSA PRIVATE KEY-----' cert_end_str = '-----END RSA PRIVATE KEY-----\n' if not certificate.startswith(cert_start_str): raise Exception("Invalid Certificate: Expected Start of Certificate to be '{}'".format(cert_start_str)) if not certificate.endswith(cert_end_str): raise Exception("Invalid Certificate: Expected End of Certificate to be '{}'".format(cert_end_str)) # Strip '-----BEGIN RSA PRIVATE KEY-----' and '-----END RSA PRIVATE KEY-----' cert_string = "".join(certificate.strip().split("\n")[1:-1]) cert_string_64 = base64.urlsafe_b64encode(cert_string.encode()) encoded_jwt = self._encode_jwt(payload, cert_string_64, header) self._raise_on_invalid_jwt_signature(encoded_jwt) return encoded_jwt def _encode_jwt(self, payload, certificate, header): return jwt.encode(payload, certificate, headers=header).decode() def _reduce_thumbprint(self, thumbprint): canonical = thumbprint.lower().replace(' ', '').replace(':', '') self._raise_on_invalid_thumbprint(canonical) return canonical def create(self, certificate, thumbprint): thumbprint = self._reduce_thumbprint(thumbprint) header = self._create_header(thumbprint) payload = self._create_payload() signed_jwt = self._sign_jwt(header, payload, certificate) return signed_jwt
class Server(object): """ High level view of a node instance. This is the object that should be created to start listening as an active node on the network. """ def __init__(self, node, ksize=20, alpha=3, storage=None): """ Create a server instance. This will start listening on the given port. Args: node: The node instance for this peer. It must contain (at minimum) an ID, public key, ip address, and port. ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper storage: An instance that implements :interface:`~dht.storage.IStorage` """ self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) self.storage = storage or ForgetfulStorage() self.node = node self.protocol = KademliaProtocol(self.node, self.storage, ksize) self.refreshLoop = LoopingCall(self.refreshTable).start(3600) def listen(self, port): """ Start listening on the given port. This is the same as calling:: reactor.listenUDP(port, server.protocol) """ return reactor.listenUDP(port, self.protocol) def refreshTable(self): """ Refresh buckets that haven't had any lookups in the last hour (per section 2.3 of the paper). """ ds = [] for id in self.protocol.getRefreshIDs(): node = Node(id) nearest = self.protocol.router.findNeighbors(node, self.alpha) spider = NodeSpiderCrawl(self.protocol, node, nearest) ds.append(spider.find()) def republishKeys(_): ds = [] # Republish keys older than one hour for keyword in self.storage.iterkeys(): for k, v in self.storage.iteritems(keyword): if self.storage.get_ttl(keyword, k) < 601200: ds.append(self.set(keyword, k, v)) return defer.gatherResults(ds).addCallback(republishKeys) def querySeed(self, seed, pubkey): """ Query an HTTP seed and return a `list` if (ip, port) `tuple` pairs. Args: seed: A `string` consisting of "ip:port" or "hostname:port" pubkey: The hex encoded public key to verify the signature on the response """ nodes = [] c = httplib.HTTPConnection(seed) c.request("GET", "/") response = c.getresponse() self.log.info("Https response from %s: %s, %s" % (seed, response.status, response.reason)) data = response.read() reread_data = data.decode("zlib") seeds = peers.PeerSeeds() try: seeds.ParseFromString(reread_data) for peer in seeds.peer_data: p = peers.PeerData() p.ParseFromString(peer) tup = (str(p.ip_address), p.port) nodes.append(tup) verify_key = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.HexEncoder) verify_key.verify(seed.signature + "".join(seeds.peer_data)) except: self.log.error("Error parsing seed response.") return nodes def bootstrappableNeighbors(self): """ Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for use as an argument to the bootstrap method. The server should have been bootstrapped already - this is just a utility for getting some neighbors and then storing them if this server is going down for a while. When it comes back up, the list of nodes can be used to bootstrap. """ neighbors = self.protocol.router.findNeighbors(self.node) return [tuple(n)[-2:] for n in neighbors] def bootstrap(self, addrs): """ Bootstrap the server by connecting to other known nodes in the network. Args: addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses are acceptable - hostnames will cause an error. """ # if the transport hasn't been initialized yet, wait a second if self.protocol.multiplexer.transport is None: return task.deferLater(reactor, 1, self.bootstrap, addrs) def initTable(results): nodes = [] for addr, result in results.items(): if result[0]: n = objects.Node() try: n.ParseFromString(result[1][0]) pubkey = n.signedPublicKey[len(n.signedPublicKey) - 32:] verify_key = nacl.signing.VerifyKey(pubkey) verify_key.verify(n.signedPublicKey) h = nacl.hash.sha512(n.signedPublicKey) pow = h[64:128] if int(pow[:6], 16) >= 50 or hexlify(n.guid) != h[:40]: raise Exception('Invalid GUID') nodes.append(Node(n.guid, addr[0], addr[1], n.signedPublicKey)) except: self.log.msg("Bootstrap node returned invalid GUID") spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha) return spider.find() ds = {} for addr in addrs: ds[addr] = self.protocol.ping((addr[0], addr[1])) return deferredDict(ds).addCallback(initTable) def inetVisibleIP(self): """ Get the internet visible IP's of this node as other nodes see it. Returns: A `list` of IP's. If no one can be contacted, then the `list` will be empty. """ def handle(results): ips = [] for result in results: if result[0]: ips.append((result[1][0], int(result[1][1]))) self.log.debug("other nodes think our ip is %s" % str(ips)) return ips ds = [] for neighbor in self.bootstrappableNeighbors(): ds.append(self.protocol.stun(neighbor)) return defer.gatherResults(ds).addCallback(handle) def get(self, keyword): """ Get a key if the network has it. Returns: :class:`None` if not found, the value otherwise. """ dkey = digest(keyword) if self.storage.get(dkey) is not None: return defer.succeed(self.storage.get(dkey)) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: self.log.warning("There are no known neighbors to get key %s" % keyword) return None spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return spider.find() def set(self, keyword, key, value): """ Set the given key/value tuple at the hash of the given keyword. All values stored in the DHT are stored as dictionaries of key/value pairs. If a value already exists for a given keyword, the new key/value pair will be appended to the dictionary. Args: keyword: a `string` keyword. The SHA1 hash of which will be used as the key when inserting in the DHT. key: the 20 byte hash of the data. value: a serialized `protos.objects.Node` object which serves as a pointer to the node storing the data. Return: True if at least one peer responded. False if the store rpc completely failed. """ self.log.debug("setting '%s' = '%s':'%s' on network" % (keyword, hexlify(key), hexlify(value))) dkey = digest(keyword) def store(nodes): self.log.info("setting '%s' on %s" % (keyword, map(str, nodes))) ds = [self.protocol.callStore(node, dkey, key, value) for node in nodes] keynode = Node(dkey) if self.node.distanceTo(keynode) < max([n.distanceTo(keynode) for n in nodes]): self.storage[dkey] = (key, value) self.log.debug("got a store request from %s, storing value" % str(self.node)) return defer.DeferredList(ds).addCallback(self._anyRespondSuccess) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: self.log.warning("There are no known neighbors to set key %s" % key) return defer.succeed(False) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) return spider.find().addCallback(store) def delete(self, keyword, key, signature): """ Delete the given key/value pair from the keyword dictionary on the network. To delete you must provide a signature covering the key that you wish to delete. It will be verified against the public key stored in the value. We use our ksize as alpha to make sure we reach as many nodes storing our value as possible. Args: keyword: the `string` keyword where the data being deleted is stored. key: the 20 byte hash of the data. signature: a signature covering the key. """ self.log.debug("deleting '%s':'%s' from the network" % (keyword, hexlify(key))) dkey = digest(keyword) def delete(nodes): self.log.info("deleting '%s' on %s" % (key, map(str, nodes))) ds = [self.protocol.callDelete(node, dkey, key, signature) for node in nodes] if self.storage.getSpecific(keyword, key) is not None: self.storage.delete(keyword, key) return defer.DeferredList(ds).addCallback(self._anyRespondSuccess) node = Node(dkey) nearest = self.protocol.router.findNeighbors(node) if len(nearest) == 0: self.log.warning("There are no known neighbors to delete key %s" % key) return defer.succeed(False) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.ksize) return spider.find().addCallback(delete) def get_node(self, guid): """ Given a guid return a `Node` object containing its ip and port or none if it's not found. Args: guid: the 20 raw bytes representing the guid. """ node_to_find = Node(guid) def check_for_node(nodes): for node in nodes: if node.id == node_to_find.id: return node return None index = self.protocol.router.getBucketFor(node_to_find) nodes = self.protocol.router.buckets[index].getNodes() for node in nodes: if node.id == node_to_find.id: return defer.succeed(node) nearest = self.protocol.router.findNeighbors(node_to_find) if len(nearest) == 0: self.log.warning("There are no known neighbors to find node %s" % node_to_find.id.encode("hex")) return defer.succeed(None) spider = NodeSpiderCrawl(self.protocol, node_to_find, nearest, self.ksize, self.alpha) return spider.find().addCallback(check_for_node) def _anyRespondSuccess(self, responses): """ Given the result of a DeferredList of calls to peers, ensure that at least one of them was contacted and responded with a Truthy result. """ for deferSuccess, result in responses: peerReached, peerResponse = result if deferSuccess and peerReached and peerResponse: return True return False def saveState(self, fname): """ Save the state of this node (the alpha/ksize/id/immediate neighbors) to a cache file with the given fname. """ data = {'ksize': self.ksize, 'alpha': self.alpha, 'id': self.node.id, 'signed_pubkey': self.node.signed_pubkey, 'neighbors': self.bootstrappableNeighbors()} if len(data['neighbors']) == 0: self.log.warning("No known neighbors, so not writing to cache.") return with open(fname, 'w') as f: pickle.dump(data, f) @classmethod def loadState(self, fname, ip_address, port, multiplexer, storage=None): """ Load the state of this node (the alpha/ksize/id/immediate neighbors) from a cache file with the given fname. """ with open(fname, 'r') as f: data = pickle.load(f) n = Node(data['id'], ip_address, port, data['signed_pubkey']) s = Server(n, data['ksize'], data['alpha'], storage=storage) s.protocol.connect_multiplexer(multiplexer) if len(data['neighbors']) > 0: s.bootstrap(data['neighbors']) return s def saveStateRegularly(self, fname, frequency=600): """ Save the state of node with a given regularity to the given filename. Args: fname: File name to save retularly to frequencey: Frequency in seconds that the state should be saved. By default, 10 minutes. """ loop = LoopingCall(self.saveState, fname) loop.start(frequency) return loop
class Lint: def __init__(self, options): self.log = Logger() if not options.workdir: raise RuntimeError, "Lint: No working directory specified." lintResult = self.__getLintResult(options.workdir) ignoreMessages = [] if hasattr(options, "ignoremessages"): optionType = type(options.ignoremessages).__name__ if optionType == "list": ignoreMessages = options.ignoremessages elif optionType == "str": ignoreMessages = options.ignoremessages.split(",") ignoreClasses = [] if hasattr(options, "ignoreclasses"): optionType = type(options.ignoreclasses).__name__ if optionType == "list": ignoreClasses = options.ignoreclasses elif optionType == "str": ignoreClasses = options.ignoreclasses.split(",") self.lintData = self.__parseLintResult(lintResult, ignoreMessages, ignoreClasses) def __getLintResult(self, workdir): self.log.info("Running lint in directory %s" %workdir) startdir = os.getcwd() os.chdir(workdir) ret,out,err = util.invokePiped("python generate.py lint") if (ret > 0): raise RuntimeError, "Lint run failed. " + err os.chdir(startdir) return out def __parseLintResult(self, text, ignoreMessages=[], ignoreClasses=[]): self.log.info("Lint parsing lint output") self.log.debug("Lint ignoring messages: %s" %repr(ignoreMessages)) self.log.debug("Lint ignoring classes: %s" %repr(ignoreClasses)) log = "" if (isinstance(text,str)): import string log = string.split(text,"\n") else: log = text data = {} for line in log: msgre = re.compile('.*\): (.*)$') msgma = msgre.match(line) msg = None if not msgma: continue msg = msgma.group(1) genericmsg = None member = None hint = None (genericmsg, member, hint) = self.__getMessage(msg) if not genericmsg: self.log.error("Lint.parseLintResult couldn't extract generic message from line:\n" + line) if (genericmsg[len(genericmsg)-3:] == " in"): genericmsg = genericmsg[0:len(genericmsg)-3] if genericmsg in ignoreMessages: continue msgid = genericmsg if (not msgid in data): data[msgid] = [] if (hint[0:2] == "! "): hint = hint[2:] info = {} info['member'] = member if (hint != ""): info['hint'] = hint info['path'] = '' info['line'] = '' pathre = re.compile('^.*([\\\/]source[\\\/].*) \(') pathma = pathre.match(line) if (pathma): info['path'] = pathma.group(1) linecolre = re.compile('.*(\(.*\)).*') linecolma = linecolre.match(line) if (linecolma): info['line'] = linecolma.group(1) ignoreClass = False for cls in ignoreClasses: classPath = cls.replace(".","/") clsre = re.compile("^.*" + classPath + ".*$") clsma = clsre.match(info['path']) if (clsma): ignoreClass = True if not ignoreClass: data[msgid].append(info) del_keys = [] for key, value in data.iteritems(): if (len(value) == 0): del_keys.append(key) for k in del_keys: del data[k] return data def __getMessage(self,fullmsg): genericmsg = None member = None hint = None msgre = re.compile("^([\w\- ]+)'([^\s]*)'([\w ]*)[\. ]*(.*)$") msgrma = msgre.match(fullmsg) if msgrma: genericmsg = msgrma.group(1) + msgrma.group(3) if (genericmsg[len(genericmsg)-1] == " "): genericmsg = genericmsg[:-1] member = msgrma.group(2) hint = msgrma.group(4) return (genericmsg, member, hint) def getResult(self): return self.lintData def getFlatResult(self): flatData = [] data = self.getResult() for message in data: for messageDetails in data[message]: flatMessage = { "message" : message, "member" : messageDetails["member"], "path" : messageDetails["path"], "line" : messageDetails["line"] } flatData.append(flatMessage) return flatData def getResultJson(self): return json.dumps(self.lintData, sort_keys=True, indent=2)
class Server(object): """ High level view of a node instance. This is the object that should be created to start listening as an active node on the network. """ def __init__(self, node, db, signing_key, ksize=20, alpha=3, storage=None): """ Create a server instance. This will start listening on the given port. Args: node: The node instance for this peer. It must contain (at minimum) an ID, public key, ip address, and port. ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper storage: An instance that implements :interface:`~dht.storage.IStorage` """ self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) self.storage = storage or ForgetfulStorage() self.node = node self.protocol = KademliaProtocol(self.node, self.storage, ksize, db, signing_key) self.refreshLoop = LoopingCall(self.refreshTable) reactor.callLater(1800, self.refreshLoop.start, 3600) def listen(self, port): """ Start listening on the given port. This is the same as calling:: reactor.listenUDP(port, server.protocol) """ return reactor.listenUDP(port, self.protocol) def refreshTable(self): """ Refresh buckets that haven't had any lookups in the last hour (per section 2.3 of the paper). """ ds = [] refresh_ids = self.protocol.getRefreshIDs() refresh_ids.append(digest(random.getrandbits(255))) # random node so we get more diversity for rid in refresh_ids: node = Node(rid) nearest = self.protocol.router.findNeighbors(node, self.alpha) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) ds.append(spider.find()) def republishKeys(_): self.log.debug("Republishing key/values...") neighbors = self.protocol.router.findNeighbors(self.node, exclude=self.node) for node in neighbors: self.protocol.transferKeyValues(node) return defer.gatherResults(ds).addCallback(republishKeys) def querySeed(self, list_seed_pubkey): """ Query an HTTP seed and return a `list` if (ip, port) `tuple` pairs. Args: Receives a list of one or more tuples Example [(seed, pubkey)] seed: A `string` consisting of "ip:port" or "hostname:port" pubkey: The hex encoded public key to verify the signature on the response """ nodes = [] if not list_seed_pubkey: self.log.error('failed to query seed {0} from ob.cfg'.format(list_seed_pubkey)) return nodes else: for sp in list_seed_pubkey: seed, pubkey = sp try: self.log.info("querying %s for peers" % seed) c = httplib.HTTPConnection(seed) c.request("GET", "/") response = c.getresponse() self.log.debug("Http response from %s: %s, %s" % (seed, response.status, response.reason)) data = response.read() reread_data = data.decode("zlib") proto = peers.PeerSeeds() proto.ParseFromString(reread_data) for peer in proto.serializedNode: n = objects.Node() n.ParseFromString(peer) tup = (str(n.nodeAddress.ip), n.nodeAddress.port) nodes.append(tup) verify_key = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.HexEncoder) verify_key.verify("".join(proto.serializedNode), proto.signature) self.log.info("%s returned %s addresses" % (seed, len(nodes))) except Exception, e: self.log.error("failed to query seed: %s" % str(e)) return nodes
class BitcoinProtocol(Protocol): def __init__(self, user_agent, inventory, subscriptions, bloom_filter, blockchain, download_listener): self.user_agent = user_agent self.inventory = inventory self.subscriptions = subscriptions self.bloom_filter = bloom_filter self.blockchain = blockchain self.download_count = 0 self.download_tracker = [0, 0] self.download_listener = download_listener self.timeouts = {} self.callbacks = {} self.state = State.CONNECTING self.version = None self.buffer = "" self.log = Logger(system=self) def connectionMade(self): """ Send the version message and start the handshake """ self.timeouts["verack"] = reactor.callLater(5, self.response_timeout, "verack") self.timeouts["version"] = reactor.callLater(5, self.response_timeout, "version") msg_version2(PROTOCOL_VERSION, self.user_agent, nStartingHeight=self.blockchain.get_height() if self.blockchain else -1).stream_serialize(self.transport) def dataReceived(self, data): self.buffer += data header = MsgHeader.from_bytes(self.buffer) if len(self.buffer) < header.msglen + 24: return try: stream = BytesIO(self.buffer) m = MsgSerializable.stream_deserialize(stream) self.buffer = stream.read() if m.command == "verack": self.timeouts["verack"].cancel() del self.timeouts["verack"] if "version" not in self.timeouts: self.on_handshake_complete() elif m.command == "version": self.version = m if m.nVersion < 70001 or m.nServices != 1: self.transport.loseConnection() self.timeouts["version"].cancel() del self.timeouts["version"] msg_verack().stream_serialize(self.transport) if self.blockchain is not None: self.to_download = self.version.nStartingHeight - self.blockchain.get_height() if "verack" not in self.timeouts: self.on_handshake_complete() elif m.command == "getdata": for item in m.inv: if item.hash in self.inventory and item.type == 1: transaction = msg_tx() transaction.tx = self.inventory[item.hash] transaction.stream_serialize(self.transport) elif m.command == "inv": for item in m.inv: # This is either an announcement of tx we broadcast ourselves or a tx we have already downloaded. # In either case we only need to callback here. if item.type == 1 and item.hash in self.subscriptions: self.subscriptions[item.hash]["callback"](item.hash) # This is the first time we are seeing this txid. Let's download it and check to see if it sends # coins to any addresses in our subscriptions. elif item.type == 1 and item.hash not in self.inventory: self.timeouts[item.hash] = reactor.callLater(5, self.response_timeout, item.hash) cinv = CInv() cinv.type = 1 cinv.hash = item.hash getdata_packet = msg_getdata() getdata_packet.inv.append(cinv) getdata_packet.stream_serialize(self.transport) # The peer announced a new block. Unlike txs, we should download it, even if we've previously # downloaded it from another peer, to make sure it doesn't contain any txs we didn't know about. elif item.type == 2 or item.type == 3: if self.state == State.DOWNLOADING: self.download_tracker[0] += 1 cinv = CInv() cinv.type = 3 cinv.hash = item.hash getdata_packet = msg_getdata() getdata_packet.inv.append(cinv) getdata_packet.stream_serialize(self.transport) if self.state != State.DOWNLOADING: self.log.debug("Peer %s:%s announced new %s %s" % (self.transport.getPeer().host, self.transport.getPeer().port, CInv.typemap[item.type], b2lx(item.hash))) elif m.command == "tx": if m.tx.GetHash() in self.timeouts: self.timeouts[m.tx.GetHash()].cancel() for out in m.tx.vout: try: addr = str(CBitcoinAddress.from_scriptPubKey(out.scriptPubKey)) except Exception: addr = None if addr in self.subscriptions: if m.tx.GetHash() not in self.subscriptions: # It's possible the first time we are hearing about this tx is following block # inclusion. If this is the case, let's make sure we include the correct number # of confirmations. in_blocks = self.inventory[m.tx.GetHash()] if m.tx.GetHash() in self.inventory else [] confirms = [] if len(in_blocks) > 0: for block in in_blocks: confirms.append(self.blockchain.get_confirmations(block)) self.subscriptions[m.tx.GetHash()] = { "announced": 0, "ann_threshold": self.subscriptions[addr][0], "confirmations": max(confirms) if len(confirms) > 0 else 0, "last_confirmation": 0, "callback": self.subscriptions[addr][1], "in_blocks": in_blocks, "tx": m.tx } self.subscriptions[addr][1](m.tx.GetHash()) if m.tx.GetHash() in self.inventory: del self.inventory[m.tx.GetHash()] elif m.command == "merkleblock": if self.blockchain is not None: self.blockchain.process_block(m.block) if self.state != State.DOWNLOADING: self.blockchain.save() # check for block inclusion of subscribed txs for match in m.block.get_matched_txs(): if match in self.subscriptions: self.subscriptions[match]["in_blocks"].append(m.block.GetHash()) else: # stick the hash here in case this is the first we are hearing about this tx. # when the tx comes over the wire after this block, we will append this hash. self.inventory[match] = [m.block.GetHash()] # run through subscriptions and callback with updated confirmations for txid in self.subscriptions: try: confirms = [] for block in self.subscriptions[txid]["in_blocks"]: confirms.append(self.blockchain.get_confirmations(block)) self.subscriptions[txid]["confirmations"] = max(confirms) self.subscriptions[txid]["callback"](txid) except Exception: pass # If we are in the middle of an initial chain download, let's check to see if we have # either reached the end of the download or if we need to loop back around and make # another get_blocks call. if self.state == State.DOWNLOADING: self.download_count += 1 percent = int((self.download_count / float(self.to_download))*100) if self.download_listener is not None: self.download_listener.progress(percent, self.download_count) self.download_listener.on_block_downloaded((self.transport.getPeer().host, self.transport.getPeer().port), header, self.to_download - self.download_count + 1) if percent == 100: if self.download_listener is not None: self.download_listener.download_complete() self.log.info("Chain download 100% complete") self.download_tracker[1] += 1 # We've downloaded every block in the inv packet and still have more to go. if (self.download_tracker[0] == self.download_tracker[1] and self.blockchain.get_height() < self.version.nStartingHeight): if self.timeouts["download"].active(): self.timeouts["download"].cancel() self.download_blocks(self.callbacks["download"]) # We've downloaded everything so let's callback to the client. elif self.blockchain.get_height() >= self.version.nStartingHeight: self.blockchain.save() self.state = State.CONNECTED self.callbacks["download"]() if self.timeouts["download"].active(): self.timeouts["download"].cancel() elif m.command == "headers": if self.timeouts["download"].active(): self.timeouts["download"].cancel() for header in m.headers: # If this node sent a block with no parent then disconnect from it and callback # on client.check_for_more_blocks. if self.blockchain.process_block(header) is None: self.blockchain.save() self.callbacks["download"]() self.transport.loseConnection() return self.download_count += 1 percent = int((self.download_count / float(self.to_download))*100) if self.download_listener is not None: self.download_listener.progress(percent, self.download_count) self.download_listener.on_block_downloaded((self.transport.getPeer().host, self.transport.getPeer().port), header, self.to_download - self.download_count + 1) if percent == 100: if self.download_listener is not None: self.download_listener.download_complete() self.log.info("Chain download 100% complete") # The headers message only comes in batches of 500 blocks. If we still have more blocks to download # loop back around and call get_headers again. if self.blockchain.get_height() < self.version.nStartingHeight: self.download_blocks(self.callbacks["download"]) else: self.blockchain.save() self.callbacks["download"]() self.state = State.CONNECTED elif m.command == "ping": msg_pong(nonce=m.nonce).stream_serialize(self.transport) else: self.log.debug("Received message %s from %s:%s" % (m.command, self.transport.getPeer().host, self.transport.getPeer().port)) if len(self.buffer) >= 24: self.dataReceived("") except Exception: traceback.print_exc() def on_handshake_complete(self): self.log.info("Connected to peer %s:%s" % (self.transport.getPeer().host, self.transport.getPeer().port)) self.load_filter() self.state = State.CONNECTED def response_timeout(self, id): if id == "download": self.callbacks["download"]() del self.timeouts[id] for t in self.timeouts.values(): if t.active(): t.cancel() if self.state != State.SHUTDOWN: self.log.warning("Peer %s:%s unresponsive, disconnecting..." % (self.transport.getPeer().host, self.transport.getPeer().port)) self.transport.loseConnection() self.state = State.SHUTDOWN def download_blocks(self, callback): if self.state == State.CONNECTING: return task.deferLater(reactor, 1, self.download_blocks, callback) if self.blockchain is not None: if self.download_listener is not None and self.download_count == 0: self.download_listener.download_started((self.transport.getPeer().host, self.transport.getPeer().port), self.to_download) self.log.info("Downloading blocks from %s:%s" % (self.transport.getPeer().host, self.transport.getPeer().port)) self.state = State.DOWNLOADING self.callbacks["download"] = callback self.timeouts["download"] = reactor.callLater(30, self.response_timeout, "download") if len(self.subscriptions) > 0: get = msg_getblocks() self.download_tracker = [0, 0] else: get = msg_getheaders() get.locator = self.blockchain.get_locator() get.stream_serialize(self.transport) def send_message(self, message_obj): if self.state == State.CONNECTING: return task.deferLater(reactor, 1, self.send_message, message_obj) message_obj.stream_serialize(self.transport) def load_filter(self): msg_filterload(filter=self.bloom_filter).stream_serialize(self.transport) def connectionLost(self, reason): self.state = State.SHUTDOWN self.log.info("Connection to %s:%s closed" % (self.transport.getPeer().host, self.transport.getPeer().port))
class RPCProtocol: """ This is an abstract class for processing and sending rpc messages. A class that implements the `MessageProcessor` interface probably should extend this as it does most of the work of keeping track of messages. """ __metaclass__ = abc.ABCMeta def __init__(self, sourceNode, router, waitTimeout=30): """ Args: sourceNode: A protobuf `Node` object containing info about this node. router: A `RoutingTable` object from dht.routing. Implies a `network.Server` object must be started first. waitTimeout: Timeout for whole messages. Note the txrudp layer has a per-packet timeout but invalid responses wont trigger it. The waitTimeout on this layer needs to be long enough to allow whole messages (ex. images) to transmit. """ self.sourceNode = sourceNode self.router = router self._waitTimeout = waitTimeout self._outstanding = {} self.log = Logger(system=self) def receive_message(self, message, sender, connection, ban_score): if message.testnet != self.multiplexer.testnet: self.log.warning( "received message from %s with incorrect network parameters." % str(connection.dest_addr)) connection.shutdown() return False if message.protoVer < PROTOCOL_VERSION: self.log.warning( "received message from %s with incompatible protocol version." % str(connection.dest_addr)) connection.shutdown() return False self.multiplexer.vendors[sender.id] = sender msgID = message.messageID if message.command == NOT_FOUND: data = None else: data = tuple(message.arguments) if msgID in self._outstanding: self._acceptResponse(msgID, data, sender) elif message.command != NOT_FOUND: ban_score.process_message(connection.dest_addr, message) self._acceptRequest(msgID, str(Command.Name(message.command)).lower(), data, sender, connection) else: ban_score.process_message(connection.dest_addr, message) def _acceptResponse(self, msgID, data, sender): if data is not None: msgargs = (b64encode(msgID), sender) self.log.debug("received response for message id %s from %s" % msgargs) else: self.log.warning("received 404 error response from %s" % sender) d = self._outstanding[msgID][0] if self._outstanding[msgID][2].active(): self._outstanding[msgID][2].cancel() d.callback((True, data)) del self._outstanding[msgID] def _acceptRequest(self, msgID, funcname, args, sender, connection): self.log.debug("received request from %s, command %s" % (sender, funcname.upper())) f = getattr(self, "rpc_%s" % funcname, None) if f is None or not callable(f): msgargs = (self.__class__.__name__, funcname) self.log.error( "%s has no callable method rpc_%s; ignoring request" % msgargs) return False if funcname == "hole_punch": f(sender, *args) else: d = defer.maybeDeferred(f, sender, *args) d.addCallback(self._sendResponse, funcname, msgID, sender, connection) d.addErrback(self._sendResponse, "bad_request", msgID, sender, connection) def _sendResponse(self, response, funcname, msgID, sender, connection): self.log.debug("sending response for msg id %s to %s" % (b64encode(msgID), sender)) m = Message() m.messageID = msgID m.sender.MergeFrom(self.sourceNode.getProto()) m.protoVer = PROTOCOL_VERSION m.testnet = self.multiplexer.testnet if response is None: m.command = NOT_FOUND else: m.command = Command.Value(funcname.upper()) if not isinstance(response, list): response = [response] for arg in response: m.arguments.append(str(arg)) m.signature = self.signing_key.sign(m.SerializeToString())[:64] connection.send_message(m.SerializeToString()) def timeout(self, node): """ This timeout is called by the txrudp connection handler. We will run through the outstanding messages and callback false on any waiting on this IP address. """ address = (node.ip, node.port) for msgID, val in self._outstanding.items(): if address == val[1]: val[0].callback((False, None)) if self._outstanding[msgID][2].active(): self._outstanding[msgID][2].cancel() del self._outstanding[msgID] self.router.removeContact(node) try: self.multiplexer[address].shutdown() except Exception: pass def rpc_hole_punch(self, sender, ip, port, relay="False"): """ A method for handling an incoming HOLE_PUNCH message. Relay the message to the correct node if it's not for us. Otherwise send a datagram to allow the other node to punch through our NAT. """ if relay == "True": self.log.debug("relaying hole punch packet to %s:%s for %s:%s" % (ip, port, sender.ip, str(sender.port))) self.hole_punch( Node(digest("null"), ip, int(port), nat_type=FULL_CONE), sender.ip, sender.port) else: self.log.debug("punching through NAT for %s:%s" % (ip, port)) # pylint: disable=W0612 for i in range(20): self.multiplexer.send_datagram("", (ip, int(port))) def __getattr__(self, name): if name.startswith("_") or name.startswith("rpc_"): return object.__getattr__(self, name) try: return object.__getattr__(self, name) except AttributeError: pass def func(node, *args): address = (node.ip, node.port) msgID = sha1(str(random.getrandbits(255))).digest() m = Message() m.messageID = msgID m.sender.MergeFrom(self.sourceNode.getProto()) m.command = Command.Value(name.upper()) m.protoVer = PROTOCOL_VERSION for arg in args: m.arguments.append(str(arg)) m.testnet = self.multiplexer.testnet m.signature = self.signing_key.sign(m.SerializeToString())[:64] data = m.SerializeToString() relay_addr = None if node.nat_type == SYMMETRIC or \ (node.nat_type == RESTRICTED and self.sourceNode.nat_type == SYMMETRIC): relay_addr = node.relay_node d = defer.Deferred() if m.command != HOLE_PUNCH: timeout = reactor.callLater(self._waitTimeout, self.timeout, node) self._outstanding[msgID] = [d, address, timeout] self.log.debug("calling remote function %s on %s (msgid %s)" % (name, address, b64encode(msgID))) self.multiplexer.send_message(data, address, relay_addr) if self.multiplexer[address].state != State.CONNECTED and \ node.nat_type == RESTRICTED and \ self.sourceNode.nat_type != SYMMETRIC and \ node.relay_node is not None: self.hole_punch( Node(digest("null"), node.relay_node[0], node.relay_node[1], nat_type=FULL_CONE), address[0], address[1], "True") self.log.debug("sending hole punch message to %s" % address[0] + ":" + str(address[1])) return d return func
def run_test(): logg = Logger() try: with open('/home/fabien/Images/blockchain1.png', 'rb') as img_file: base64.b64encode(img_file.read()) except (IOError, TypeError): pass try: with open('/home/fabien/Images/matryoshka.png', 'rb') as img_file: base64.b64encode(img_file.read()) except (IOError, TypeError): pass username = "******" password_hash = hashlib.sha1("pikcio_node_password").digest().enode('hex') client_api = ClientAPI(username=username, password=password_hash) access = client_api.get_access_token() logg.debug("\n===Access :\n {0} \n===\n".format(access)) ########################################################################### ############################################################ # # # UPDATE TEST 31/08/18 # # # ############################################################ ################################## # Everything related to the user # ################################## # OK : ''' json_update_password = { "current_password": "******", "password": "******", "retype_password": "******" } result = client_api.update_password(json_update_password) ''' ''' json_get_reset_token_forgotten_password = { "question_id": 1, "answer": "answer1" } result = client_api.get_reset_token_forgotten_password( json_get_reset_token_forgotten_password ) ''' ''' json_reset_password = { "reset_token": "4525019d46b3c424d77917c9b2e1da781b9cec2e", "new_password": "******", "confirm_password": "******" } result = client_api.reset_password(json_reset_password) ''' result = client_api.get_user_avatar() ''' json_set_user_avatar = { "avatar": { "data": file_data, "value": "sangoku_api.png" } } result = client_api.set_user_avatar(json_set_user_avatar) ''' # result = client_api.get_user_profile() ''' json_update_user_profile = { "profile": { "children": {'value': 'azert', 'shared': False}, "birth_date": {'value': '2', 'shared': True}, "country": {'value': 'France', 'shared': False}, "my new item": {'value': 'LE CRES', 'shared': False} } } result = client_api.update_user_profile(json_update_user_profile) ''' # result = client_api.get_background() # result = client_api.get_tci() ''' json_delete_custom_profile_item = {"item": "re test"} result = client_api.delete_custom_profile_item( json_delete_custom_profile_item ) ''' ''' json_upload_data = { "data": "Jana kramer", "metadata": "say_my_name", "context": "Professional", "source": "API" } result = client_api.upload_data(json_upload_data) ''' # TO BE TESTED : # not included in the current version : ''' json_register = { "username": "", "firstname": "", "lastname": "", "email": "" } client_api.register() ''' ''' json_login = { "username": "******", "password": "******" } client_api.login(json_login) ''' # client_api.logout() # result = client_api.set_background('data_45') # result = client_api.import_profile() # result = client_api.export_profile() ########################################################################### ################################## # Everything related to contacts # ################################## # OK : # result = client_api.find_user( # query='41f91225e521b2bb03f7680f26a8b366cb134c3e' # ) ''' json_add_contact = { "matr_id": "414d544d385356486b50346a434668744e73596754393574573872444" "34d7a463375", "username": "******" } result = client_api.add_contact(json_add_contact) ''' # result = client_api.get_contacts() # result = client_api.get_contact_profile( # matr_id='414d544d385356486b50346a57457387244434d7a463375' # ) ''' json_add_contact = { "matr_id": "414d544d385356486b50346a57457387244434d7a463375", "keep_files": True } result = client_api.remove_contact(json_add_contact) ''' # result = client_api.accept_contact_request( # matr_id="414d544d385356486b50346a57457387244434d7a463375" # ) # result = client_api.reject_contact_request( # matr_id="414d544d385356486b50346a57457387244434d7a463375" # ) ''' json_block_contact = { "matr_id": "414d544d385356486b50346a57457387244434d7a463375", "keep_files": False } result = client_api.block_contact(json_block_contact) ''' # result = client_api.unblock_contact( # matr_id="414d544d385356486b50346a57457387244434d7a463375" # ) ''' json_block_contact = { "matr_id": "414d544d385356486b50346a434668744e73596754393574573872444" "34d7a463375", "action": "remove" } result = client_api.manage_replicant(json_block_contact) ''' # TO BE TESTED : # not included in the current version : # client_api.import_contacts() # client_api.invite_someone() # result = client_api.get_notifications_online() ########################################################################### ######################################### # Everything related to Pikcio messages # ######################################### # OK : ''' json_get_pikcio_messages = { "category": "received", "limit": 0, "date": "2017-04-01 12:12:12", "filter": "after" } result = client_api.get_pikcio_messages(json_get_pikcio_messages) # result = client_api.get_pikcio_message( # msg_id='211197ad9109c3cf0f5793651e38f7dae5b5d28a' # ) # result = client_api.delete_pikcio_message( # msg_id='211197ad9109c3cf0f5793651e38f7dae5b5d28a' # ) ''' ''' json_send_pikcio_message = { "content": "This is a text content of a Pikcio message", "subject": "TEST from API", "lifetime": '30', "certification": False, "external_receivers": [ {"email": "dev@matchupbox", "name": "Support Matchupbox"} ], "receivers": [ "414d544d385356486b50346a434668744e7359675439357457387244434d7a46" "3375" ] } result = client_api.send_pikcio_message(json_send_pikcio_message) ''' # result = client_api.set_pikcio_message_to_read( # msg_id='bfd927a239f2d8e61a9133e1fbe2b016d2de619b' # ) # result = client_api.pikcio_message_request_certification( # msg_id='bfd927a239f2d8e61a9133e1fbe2b016d2de619b' # ) ########################################################################### ####################################### # Everything related to chat messages # ####################################### # OK : ''' json_get_conversation = { "chat": { "matr_id": "414d544d385356486b50346a434668744e7359675439357457387" "244434d7a463375", "date": "2018-01-01", "filter": "after", "limit": 1 } } result = client_api.get_chat_conversation(json_get_conversation) ''' ''' json_send_chat_message = { "receivers": [ { 'matr_id': '415133767636324c6d594c476246375138623943527a32' '326972453566533933616b' }, { 'matr_id': '414d544d385356486b50346a434668744e735967543935' '7457387244434d7a463375' } ], "message": "Test from API NEW VERSION", "certify": False } result = client_api.send_chat_message(json_send_chat_message) ''' # result = client_api.delete_chat_message( # msg_id='0d1d588ad8393d7796066a67420641efb8776f71' # ) # result = client_api.get_chat_message_status( # msg_id='450f1a5002cbc8071dbd8e47e86f4bfed7509df5' # ) # result = client_api.set_chat_message_to_read( # msg_id='efa996a5a6216c696d112e1aa82578df05798fdd' # ) # KO : # result = client_api.delete_chat_conversation( # matr_id="414d544d385356486b50346a57457387244434d7a463375" # ) ########################################################################### ####################################### # Everything related to file messages # ####################################### # OK: ''' json_get_file_messages = { "file": { "matr_id": '414d544d385356486b50346a434668744e7359675439357457' '387244434d7a463375', "date": '2016-04-18 09:06:16', "filter": "after", "limit": 2 } } result = client_api.get_file_messages(json_get_file_messages) ''' ''' json_send_file_message = { "receivers": [ { 'matr_id': '415133767636324c6d594c476246375138623943527a32' '326972453566533933616b' }, { 'matr_id': '414d544d385356486b50346a434668744e735967543935' '7457387244434d7a463375' } ], "file_name": 'testingAPI.png', "file_content": file_data, "certify": True } result = client_api.send_file_message(json_send_file_message) ''' # result = client_api.set_file_message_to_read( # msg_id='459e1debe9310630e001ccd8acbaa2048f64c355' # ) # result = client_api.delete_file_message( # msg_id='459e1debe9310630e001ccd8acbaa2048f64c355' # ) # MISSING PikcioChain.py : # client_api.get_file_message_meta_data() # client_api.get_file_message_matr_msg_id() ########################################################################### ####################################### # Everything related to wall messages # ####################################### # OK : ''' json_get_wall_messages = { 'wall': { "limit": 2, "date": "2017-01-28 14:14:14", "filter": "after" } } result = client_api.get_wall_messages(json_get_wall_messages) ''' ''' json_send_wall_file_message = { "file": { "data": file_data, "name": "test_wall from API.jpg" } } result = client_api.send_wall_message(json_send_wall_file_message) ''' ''' json_send_wall_text_message = { "message": { "data": "test from api" } } result = client_api.send_wall_message(json_send_wall_text_message) ''' # result = client_api.delete_wall_message( # message_id='a5d6e553c251c26680f8d887e815e0a59af5866f' # ) # result = client_api.get_wall_message_comments( # message_id='09152ea2963136d5ea5be98a79e8908f4dbf8dda' # ) # to be tested : # not included in the current version # todo : all features concerning public wall (send, retrieve, ...) ########################################################################### ######################################### # Everything related to file management # ######################################### # OK : ''' json_upload_file = { "files": [ { "file_name": "", "file_content": file_data, "tags": "api blockchain, test", "contexts": "" }, { "file_name": "matryoshka.png", "file_content": file_data2, "tags": "", "contexts": "" } ], "folder_name": "APIFolder" } result = client_api.upload_file(json_upload_file) ''' ''' json_delete_file = { "file_id": 1591, "check": False } result = client_api.delete_file(json_delete_file) ''' ''' json_get_files_info = { "folder_name": "APIFolder", "limit": 2 } result = client_api.get_files_info(json_get_files_info) ''' # result = client_api.get_file_info('8') # result = client_api.get_folders() # result = client_api.encrypt_file('6') # result = client_api.decrypt_file('8') # result = client_api.certify_file('8') # result = client_api.request_file_history('8') # KO : # result = client_api.delete_folder("APIFolder") # result = client_api.move_file(data_43) ########################################################################### ################################## # Everything related to Settings # ################################## # OK : # result = client_api.report_bug("test API") # to be tested : # not included in the current version : # result = client_api.edit_network() # result = client_api.edit_message_lifetime() # result = client_api.create_api() # result = client_api.delete_api() # result = client_api.get_api_info() # result = client_api.get_api_config_file() # result = client_api.get_api_qr_code() # result = client_api.toggle_popups() ########################################################################### ################################ # Everything related to Groups # ################################ # ok : # to be tested : # not included in the current version : # result = client_api.create_group(data_16) # result = client_api.delete_group(name="TestAPI") # result = client_api.list_group() # result = client_api.get_group_info(data_18) # result = client_api.add_contact_to_group(data_17) # result = client_api.remove_contact_from_group(data_17) # result = client_api.send_group_file_message(data_19) # result = client_api.send_group_chat_message(data_20) # result = client_api.delete_group_message( # msg_id="73ba804f503b8c4f7c9ccf35fb369ec860871496" # ) # result = client_api.get_group_messages(name="TestAPI") ########################################################################### ################################ # Everything related to Agenda # ################################ # ok : # to be tested : # not included in the current version : # client_api.get_agenda_events(data_24) # client_api.get_event("d368b18421fb30a45d8b262f526d7aa31d57526a") # client_api.create_event(data_22) # client_api.update_event(data_23) # client_api.delete_event(event_id=) # client_api.reject_event(event_id=) # client_api.accept_event(event_id=) ########################################################################### ############################### # Everything related to Email # ############################### # ok : # to be tested : # not included in the current version : # client_api.edit_emails_frequency(data=) # client_api.link_emails(data=) ########################################################################### ################################ # Everything related to Wallet # ################################ # ok : ''' json_make_payment = { "matr_id": "40b3551fe8cd576e710d613f79c199360533269f", "amount": "1.125" } result = client_api.make_payment(json_make_payment) ''' # result = client_api.update_wallet() # result = client_api.get_transactions() # to be tested : # not included in the current version : ########################################################################### ######################################### # Everything related to Generic Message # ######################################### # ok : ''' custom_content = { 'brand': 'BMW', 'model': 'M6' } json_send_generic_message = { 'type': 'Ouioui', 'scope': 'film', 'action': 'send', 'content': custom_content, 'receivers': ['1a3dc918c938a22ab4b3d8ab7d9fae55cdb2b8cb'], 'certify': False, 'attachments': ['6'] } custom_content_msf_create_case = { 'type': 'MSF', 'scope': 'medical_case', 'action': 'create', 'receivers': ['b71b9522f2050ad93f064e66738bbabc33225cd5'], 'certify': False, 'content': { 'type': 'Patient-related clinical query (to a specific patient)', 'subject': 'Cancer', 'language': 'EN', 'localization': 'Dire dawa, Ethiopia', 'patient_name': 'Lilian', 'patient_lastname': 'Laslandes', 'patient_age': '45', 'gender': 'male', 'priority': 'critical', 'referrer_id': '2', 'complaint': 'Lorem Ipsum is simply dummy text of the printing and' ' typesetting industry.', 'history': 'and a search for lorem ipsum will uncover many web ' 'sites still in their infancy.', 'past': 'The standard chunk of Lorem Ipsum used since the 1500s' }, 'attachments': [] } # result = client_api.send_generic_message(json_send_generic_message) ''' ''' json_get_generic_messages1 = { 'type': 'MSF', 'scope': 'medical_case', 'action': '', 'is_read': True } json_get_generic_messages2 = { 'type': 'Ouioui', 'scope': 'film', 'action': '', 'limit': 2, 'date': '', 'filter_date': '', 'is_read': True } result = client_api.get_generic_messages(json_get_generic_messages1) ''' # result = client_api.delete_generic_messages( # message_id='f798701d1249944e8cc8214b5b5e353fd50511ff' # ) # result = client_api.get_generic_message_status( # message_id='b8b5e24dcf446ca6faeda4a944f41a08a117740c' # ) # to be tested : # not included in the current version : ########################################################################### if isinstance(result, str): logg.debug("\n1===\n{0} \n===\n".format(result)) else: logg.debug("\n2===\n{0} \n===\n".format(result.content))
class MarketProtocol(RPCProtocol): implements(MessageProcessor) def __init__(self, node_proto, router, signing_key): self.router = router RPCProtocol.__init__(self, node_proto, router) self.log = Logger(system=self) self.multiplexer = None self.hashmap = HashMap() self.signing_key = signing_key self.listeners = [] self.handled_commands = [GET_CONTRACT, GET_IMAGE, GET_PROFILE, GET_LISTINGS, GET_USER_METADATA, GET_CONTRACT_METADATA, FOLLOW, UNFOLLOW, GET_FOLLOWERS, GET_FOLLOWING, NOTIFY, MESSAGE] def connect_multiplexer(self, multiplexer): self.multiplexer = multiplexer def add_listener(self, listener): self.listeners.append(listener) def rpc_get_contract(self, sender, contract_hash): self.log.info("Looking up contract ID %s" % contract_hash.encode('hex')) self.router.addContact(sender) try: with open(self.hashmap.get_file(contract_hash), "r") as filename: contract = filename.read() return [contract] except Exception: self.log.warning("Could not find contract %s" % contract_hash.encode('hex')) return ["None"] def rpc_get_image(self, sender, image_hash): self.log.info("Looking up image with hash %s" % image_hash.encode('hex')) self.router.addContact(sender) try: with open(self.hashmap.get_file(image_hash), "r") as filename: image = filename.read() return [image] except Exception: self.log.warning("Could not find image %s" % image_hash.encode('hex')) return ["None"] def rpc_get_profile(self, sender): self.log.info("Fetching profile") self.router.addContact(sender) try: proto = Profile().get(True) return [proto, self.signing_key.sign(proto)[:64]] except Exception: self.log.error("Unable to load the profile") return ["None"] def rpc_get_user_metadata(self, sender): self.log.info("Fetching metadata") self.router.addContact(sender) try: proto = Profile().get(False) m = Metadata() m.name = proto.name m.handle = proto.handle m.short_description = proto.short_description m.avatar_hash = proto.avatar_hash m.nsfw = proto.nsfw return [m.SerializeToString(), self.signing_key.sign(m.SerializeToString())[:64]] except Exception: self.log.error("Unable to get the profile metadata") return ["None"] def rpc_get_listings(self, sender): self.log.info("Fetching listings") self.router.addContact(sender) try: p = Profile().get() l = Listings() l.ParseFromString(ListingsStore().get_proto()) l.handle = p.handle l.avatar_hash = p.avatar_hash return [l.SerializeToString(), self.signing_key.sign(l.SerializeToString())[:64]] except Exception: self.log.warning("Could not find any listings in the database") return ["None"] def rpc_get_contract_metadata(self, sender, contract_hash): self.log.info("Fetching metadata for contract %s" % hexlify(contract_hash)) self.router.addContact(sender) try: proto = ListingsStore().get_proto() l = Listings() l.ParseFromString(proto) for listing in l.listing: if listing.contract_hash == contract_hash: ser = listing.SerializeToString() return [ser, self.signing_key.sign(ser)[:64]] except Exception: self.log.warning("Could not find metadata for contract %s" % hexlify(contract_hash)) return ["None"] def rpc_follow(self, sender, proto, signature): self.log.info("Follow request from %s" % sender.id.encode("hex")) self.router.addContact(sender) try: verify_key = nacl.signing.VerifyKey(sender.signed_pubkey[64:]) verify_key.verify(proto, signature) f = Followers.Follower() f.ParseFromString(proto) if f.guid != sender.id: raise Exception('GUID does not match sending node') if f.following != self.proto.guid: raise Exception('Following wrong node') f.signature = signature FollowData().set_follower(f) proto = Profile().get(False) m = Metadata() m.name = proto.name m.handle = proto.handle m.avatar_hash = proto.avatar_hash m.nsfw = proto.nsfw return ["True", m.SerializeToString(), self.signing_key.sign(m.SerializeToString())[:64]] except Exception: self.log.warning("Failed to validate follower") return ["False"] def rpc_unfollow(self, sender, signature): self.log.info("Unfollow request from %s" % sender.id.encode("hex")) self.router.addContact(sender) try: verify_key = nacl.signing.VerifyKey(sender.signed_pubkey[64:]) verify_key.verify("unfollow:" + self.proto.guid, signature) f = FollowData() f.delete_follower(sender.id) return ["True"] except Exception: self.log.warning("Failed to validate follower signature") return ["False"] def rpc_get_followers(self, sender): self.log.info("Fetching followers list from db") self.router.addContact(sender) ser = FollowData().get_followers() if ser is None: return ["None"] else: return [ser, self.signing_key.sign(ser)[:64]] def rpc_get_following(self, sender): self.log.info("Fetching following list from db") self.router.addContact(sender) ser = FollowData().get_following() if ser is None: return ["None"] else: return [ser, self.signing_key.sign(ser)[:64]] def rpc_notify(self, sender, message, signature): if len(message) <= 140 and FollowData().is_following(sender.id): try: verify_key = nacl.signing.VerifyKey(sender.signed_pubkey[64:]) verify_key.verify(message, signature) except Exception: return ["False"] self.log.info("Received a notification from %s" % sender) self.router.addContact(sender) for listener in self.listeners: try: verifyObject(NotificationListener, listener) listener.notify(sender.id, message) except DoesNotImplement: pass return ["True"] else: return ["False"] def rpc_message(self, sender, pubkey, encrypted): try: box = Box(PrivateKey(self.signing_key.encode(nacl.encoding.RawEncoder)), PublicKey(pubkey)) plaintext = box.decrypt(encrypted) p = Plaintext_Message() p.ParseFromString(plaintext) signature = p.signature p.ClearField("signature") verify_key = nacl.signing.VerifyKey(p.signed_pubkey[64:]) verify_key.verify(p.SerializeToString(), signature) h = nacl.hash.sha512(p.signed_pubkey) pow_hash = h[64:128] if int(pow_hash[:6], 16) >= 50 or hexlify(p.sender_guid) != h[:40] or p.sender_guid != sender.id: raise Exception('Invalid guid') self.log.info("Received a message from %s" % sender) self.router.addContact(sender) for listener in self.listeners: try: verifyObject(MessageListener, listener) listener.notify(p, signature) except DoesNotImplement: pass return ["True"] except Exception: self.log.error("Received invalid message from %s" % sender) return ["False"] def callGetContract(self, nodeToAsk, contract_hash): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_contract(address, contract_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetImage(self, nodeToAsk, image_hash): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_image(address, image_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetProfile(self, nodeToAsk): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_profile(address) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetUserMetadata(self, nodeToAsk): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_user_metadata(address) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetListings(self, nodeToAsk): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_listings(address) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetContractMetadata(self, nodeToAsk, contract_hash): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_contract_metadata(address, contract_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callFollow(self, nodeToAsk, proto, signature): address = (nodeToAsk.ip, nodeToAsk.port) d = self.follow(address, proto, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callUnfollow(self, nodeToAsk, signature): address = (nodeToAsk.ip, nodeToAsk.port) d = self.unfollow(address, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetFollowers(self, nodeToAsk): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_followers(address) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetFollowing(self, nodeToAsk): address = (nodeToAsk.ip, nodeToAsk.port) d = self.get_following(address) return d.addCallback(self.handleCallResponse, nodeToAsk) def callNotify(self, nodeToAsk, message, signature): address = (nodeToAsk.ip, nodeToAsk.port) d = self.notify(address, message, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callMessage(self, nodeToAsk, ehemeral_pubkey, ciphertext): address = (nodeToAsk.ip, nodeToAsk.port) d = self.message(address, ehemeral_pubkey, ciphertext) return d.addCallback(self.handleCallResponse, nodeToAsk) def handleCallResponse(self, result, node): """ If we get a response, add the node to the routing table. If we get no response, make sure it's removed from the routing table. """ if result[0]: self.log.info("got response from %s, adding to router" % node) self.router.addContact(node) else: self.log.debug("no response from %s, removing from router" % node) self.router.removeContact(node) return result def __iter__(self): return iter(self.handled_commands)
class RPCProtocol: """ This is an abstract class for processing and sending rpc messages. A class that implements the `MessageProcessor` interface probably should extend this as it does most of the work of keeping track of messages. """ __metaclass__ = abc.ABCMeta def __init__(self, proto, router, waitTimeout=5): """ Args: proto: A protobuf `Node` object containing info about this node. router: A `RoutingTable` object from dht.routing. Implies a `network.Server` object must be started first. waitTimeout: Consider it a connetion failure if no response within this time window. noisy: Whether or not to log the output for this class. testnet: The network parameters to use. """ self.proto = proto self.router = router self._waitTimeout = waitTimeout self._outstanding = {} self.log = Logger(system=self) def receive_message(self, datagram, connection): m = Message() try: m.ParseFromString(datagram) sender = node.Node(m.sender.guid, m.sender.ip, m.sender.port, m.sender.signedPublicKey, m.sender.vendor) except Exception: # If message isn't formatted property then ignore self.log.warning("received unknown message from %s, ignoring" % str(connection.dest_addr)) return False if m.testnet != self.multiplexer.testnet: self.log.warning("received message from %s with incorrect network parameters." % str(connection.dest_addr)) connection.shutdown() return False if m.protoVer < PROTOCOL_VERSION: self.log.warning("received message from %s with incompatible protocol version." % str(connection.dest_addr)) connection.shutdown() return False # Check that the GUID is valid. If not, ignore if self.router.isNewNode(sender): try: pubkey = m.sender.signedPublicKey[len(m.sender.signedPublicKey) - 32:] verify_key = nacl.signing.VerifyKey(pubkey) verify_key.verify(m.sender.signedPublicKey) h = nacl.hash.sha512(m.sender.signedPublicKey) pow_hash = h[64:128] if int(pow_hash[:6], 16) >= 50 or hexlify(m.sender.guid) != h[:40]: raise Exception('Invalid GUID') except Exception: self.log.warning("received message from sender with invalid GUID, ignoring") connection.shutdown() return False if m.sender.vendor: self.db.VendorStore().save_vendor(m.sender.guid, m.sender.ip, m.sender.port, m.sender.signedPublicKey) msgID = m.messageID if m.command == NOT_FOUND: data = None else: data = tuple(m.arguments) if msgID in self._outstanding: self._acceptResponse(msgID, data, sender) elif m.command != NOT_FOUND: self._acceptRequest(msgID, str(Command.Name(m.command)).lower(), data, sender, connection) def _acceptResponse(self, msgID, data, sender): if data is not None: msgargs = (b64encode(msgID), sender) self.log.debug("received response for message id %s from %s" % msgargs) else: self.log.warning("received 404 error response from %s" % sender) d, timeout = self._outstanding[msgID] timeout.cancel() d.callback((True, data)) del self._outstanding[msgID] def _acceptRequest(self, msgID, funcname, args, sender, connection): self.log.debug("received request from %s, command %s" % (sender, funcname.upper())) f = getattr(self, "rpc_%s" % funcname, None) if f is None or not callable(f): msgargs = (self.__class__.__name__, funcname) self.log.error("%s has no callable method rpc_%s; ignoring request" % msgargs) return False if funcname == "hole_punch": f(sender, *args) else: d = defer.maybeDeferred(f, sender, *args) d.addCallback(self._sendResponse, funcname, msgID, sender, connection) def _sendResponse(self, response, funcname, msgID, sender, connection): self.log.debug("sending response for msg id %s to %s" % (b64encode(msgID), sender)) m = Message() m.messageID = msgID m.sender.MergeFrom(self.proto) m.protoVer = PROTOCOL_VERSION m.testnet = self.multiplexer.testnet if response is None: m.command = NOT_FOUND else: m.command = Command.Value(funcname.upper()) for arg in response: m.arguments.append(str(arg)) data = m.SerializeToString() connection.send_message(data) def _timeout(self, msgID, address): """ If a message times out we are first going to try hole punching because the node may be behind a restricted NAT. If it is successful, the original should get through. This timeout will only fire if the hole punching fails. """ # pylint: disable=pointless-string-statement """ Hole punching disabled for now seed = SEED_NODE_TESTNET if self.multiplexer.testnet else SEED_NODE if not hp and self.multiplexer.ip_address[0] != seed[0]: args = (address[0], address[1], b64encode(msgID)) self.log.debug("did not receive reply from %s:%s for msgID %s, trying hole punching..." % args) self.hole_punch(seed, address[0], address[1], "True") timeout = reactor.callLater(self._waitTimeout, self._timeout, msgID, address, True) self._outstanding[msgID][1] = timeout else: """ args = (b64encode(msgID), self._waitTimeout) self.log.warning("did not receive reply for msg id %s within %i seconds" % args) self._outstanding[msgID][0].callback((False, None)) del self._outstanding[msgID] self.multiplexer[address].shutdown() def rpc_hole_punch(self, sender, ip, port, relay="False"): """ A method for handling an incoming HOLE_PUNCH message. Relay the message to the correct node if it's not for us. Otherwise sent a datagram to allow the other node to punch through our NAT. """ if relay == "True": self.hole_punch((ip, int(port)), sender.ip, sender.port) else: self.log.debug("punching through NAT for %s:%s" % (ip, port)) self.multiplexer.send_datagram(" ", (ip, int(port))) def _get_waitTimeout(self, command): if command == GET_IMAGE or command == GET_CONTRACT: return 100 else: return self._waitTimeout def __getattr__(self, name): if name.startswith("_") or name.startswith("rpc_"): return object.__getattr__(self, name) try: return object.__getattr__(self, name) except AttributeError: pass def func(address, *args): msgID = sha1(str(random.getrandbits(255))).digest() m = Message() m.messageID = msgID m.sender.MergeFrom(self.proto) m.command = Command.Value(name.upper()) m.protoVer = PROTOCOL_VERSION for arg in args: m.arguments.append(str(arg)) m.testnet = self.multiplexer.testnet data = m.SerializeToString() d = defer.Deferred() timeout = reactor.callLater(self._get_waitTimeout(m.command), self._timeout, msgID, address) self._outstanding[msgID] = [d, timeout] self.multiplexer.send_message(data, address) self.log.debug("calling remote function %s on %s (msgid %s)" % (name, address, b64encode(msgID))) return d return func
class RPCProtocol: """ This is an abstract class for processing and sending rpc messages. A class that implements the `MessageProcessor` interface probably should extend this as it does most of the work of keeping track of messages. """ __metaclass__ = abc.ABCMeta def __init__(self, sourceNode, router, waitTimeout=15): """ Args: sourceNode: A protobuf `Node` object containing info about this node. router: A `RoutingTable` object from dht.routing. Implies a `network.Server` object must be started first. waitTimeout: Timeout for whole messages. Note the txrudp layer has a per-packet timeout but invalid responses wont trigger it. The waitTimeout on this layer needs to be long enough to allow whole messages (ex. images) to transmit. """ self.sourceNode = sourceNode self.router = router self._waitTimeout = waitTimeout self._outstanding = {} self.log = Logger(system=self) def receive_message(self, message, sender, connection, ban_score): if message.testnet != self.multiplexer.testnet: self.log.warning("received message from %s with incorrect network parameters." % str(connection.dest_addr)) connection.shutdown() return False if message.protoVer < PROTOCOL_VERSION: self.log.warning("received message from %s with incompatible protocol version." % str(connection.dest_addr)) connection.shutdown() return False self.multiplexer.vendors[sender.id] = sender msgID = message.messageID if message.command == NOT_FOUND: data = None else: data = tuple(message.arguments) if msgID in self._outstanding: self._acceptResponse(msgID, data, sender) elif message.command != NOT_FOUND: # ban_score.process_message(message) self._acceptRequest(msgID, str(Command.Name(message.command)).lower(), data, sender, connection) def _acceptResponse(self, msgID, data, sender): if data is not None: msgargs = (b64encode(msgID), sender) self.log.debug("received response for message id %s from %s" % msgargs) else: self.log.warning("received 404 error response from %s" % sender) d = self._outstanding[msgID][0] if self._outstanding[msgID][2].active(): self._outstanding[msgID][2].cancel() d.callback((True, data)) del self._outstanding[msgID] def _acceptRequest(self, msgID, funcname, args, sender, connection): self.log.debug("received request from %s, command %s" % (sender, funcname.upper())) f = getattr(self, "rpc_%s" % funcname, None) if f is None or not callable(f): msgargs = (self.__class__.__name__, funcname) self.log.error("%s has no callable method rpc_%s; ignoring request" % msgargs) return False if funcname == "hole_punch": f(sender, *args) else: d = defer.maybeDeferred(f, sender, *args) d.addCallback(self._sendResponse, funcname, msgID, sender, connection) d.addErrback(self._sendResponse, "bad_request", msgID, sender, connection) def _sendResponse(self, response, funcname, msgID, sender, connection): self.log.debug("sending response for msg id %s to %s" % (b64encode(msgID), sender)) m = Message() m.messageID = msgID m.sender.MergeFrom(self.sourceNode.getProto()) m.protoVer = PROTOCOL_VERSION m.testnet = self.multiplexer.testnet if response is None: m.command = NOT_FOUND else: m.command = Command.Value(funcname.upper()) if not isinstance(response, list): response = [response] for arg in response: m.arguments.append(str(arg)) m.signature = self.signing_key.sign(m.SerializeToString())[:64] connection.send_message(m.SerializeToString()) def timeout(self, node): """ This timeout is called by the txrudp connection handler. We will run through the outstanding messages and callback false on any waiting on this IP address. """ address = (node.ip, node.port) for msgID, val in self._outstanding.items(): if address == val[1]: val[0].callback((False, None)) if self._outstanding[msgID][2].active(): self._outstanding[msgID][2].cancel() del self._outstanding[msgID] self.router.removeContact(node) try: self.multiplexer[address].shutdown() except Exception: pass def rpc_hole_punch(self, sender, ip, port, relay="False"): """ A method for handling an incoming HOLE_PUNCH message. Relay the message to the correct node if it's not for us. Otherwise send a datagram to allow the other node to punch through our NAT. """ if relay == "True": self.log.debug("relaying hole punch packet to %s:%s for %s:%s" % (ip, port, sender.ip, str(sender.port))) self.hole_punch(Node(digest("null"), ip, int(port), nat_type=FULL_CONE), sender.ip, sender.port) else: self.log.debug("punching through NAT for %s:%s" % (ip, port)) # pylint: disable=W0612 for i in range(20): self.multiplexer.send_datagram("", (ip, int(port))) def __getattr__(self, name): if name.startswith("_") or name.startswith("rpc_"): return object.__getattr__(self, name) try: return object.__getattr__(self, name) except AttributeError: pass def func(node, *args): msgID = sha1(str(random.getrandbits(255))).digest() m = Message() m.messageID = msgID m.sender.MergeFrom(self.sourceNode.getProto()) m.command = Command.Value(name.upper()) m.protoVer = PROTOCOL_VERSION for arg in args: m.arguments.append(str(arg)) m.testnet = self.multiplexer.testnet m.signature = self.signing_key.sign(m.SerializeToString())[:64] data = m.SerializeToString() address = (node.ip, node.port) relay_addr = None if node.nat_type == SYMMETRIC or \ (node.nat_type == RESTRICTED and self.sourceNode.nat_type == SYMMETRIC): relay_addr = node.relay_node d = defer.Deferred() if m.command != HOLE_PUNCH: timeout = reactor.callLater(self._waitTimeout, self.timeout, node) self._outstanding[msgID] = [d, address, timeout] self.log.debug("calling remote function %s on %s (msgid %s)" % (name, address, b64encode(msgID))) self.multiplexer.send_message(data, address, relay_addr) if self.multiplexer[address].state != State.CONNECTED and \ node.nat_type == RESTRICTED and \ self.sourceNode.nat_type != SYMMETRIC: self.hole_punch(Node(digest("null"), node.relay_node[0], node.relay_node[1], nat_type=FULL_CONE), address[0], address[1], "True") self.log.debug("sending hole punch message to %s" % address[0] + ":" + str(address[1])) return d return func
class KademliaProtocol(RPCProtocol): implements(MessageProcessor) def __init__(self, sourceNode, storage, ksize, database, signing_key): self.ksize = ksize self.router = RoutingTable(self, ksize, sourceNode) self.storage = storage self.sourceNode = sourceNode self.multiplexer = None self.db = database self.signing_key = signing_key self.log = Logger(system=self) self.handled_commands = [PING, STUN, STORE, DELETE, FIND_NODE, FIND_VALUE, HOLE_PUNCH, INV, VALUES] self.recent_transfers = set() RPCProtocol.__init__(self, sourceNode, self.router) def connect_multiplexer(self, multiplexer): self.multiplexer = multiplexer def getRefreshIDs(self): """ Get ids to search for to keep old buckets up to date. """ ids = [] for bucket in self.router.getLonelyBuckets(): ids.append(random.randint(*bucket.range)) return ids def rpc_stun(self, sender): self.addToRouter(sender) return [sender.ip, str(sender.port)] def rpc_ping(self, sender): self.addToRouter(sender) return [self.sourceNode.getProto().SerializeToString()] def rpc_store(self, sender, keyword, key, value, ttl): self.addToRouter(sender) self.log.debug("got a store request from %s, storing value" % str(sender)) if len(keyword) == 20 and len(key) <= 33 and len(value) <= 2100 and int(ttl) <= 604800: self.storage[keyword] = (key, value, int(ttl)) return ["True"] else: return ["False"] def rpc_delete(self, sender, keyword, key, signature): self.addToRouter(sender) value = self.storage.getSpecific(keyword, key) if value is not None: # Try to delete a message from the dht if keyword == digest(sender.id): try: verify_key = nacl.signing.VerifyKey(sender.pubkey) verify_key.verify(key, signature) self.storage.delete(keyword, key) return ["True"] except Exception: return ["False"] # Or try to delete a pointer else: try: node = objects.Node() node.ParseFromString(value) pubkey = node.publicKey try: verify_key = nacl.signing.VerifyKey(pubkey) verify_key.verify(key, signature) self.storage.delete(keyword, key) return ["True"] except Exception: return ["False"] except Exception: pass return ["False"] def rpc_find_node(self, sender, key): self.log.debug("finding neighbors of %s in local table" % key.encode('hex')) self.addToRouter(sender) node = Node(key) nodeList = self.router.findNeighbors(node, exclude=sender) ret = [] if self.sourceNode.id == key: ret.append(self.sourceNode.getProto().SerializeToString()) for n in nodeList: ret.append(n.getProto().SerializeToString()) return ret def rpc_find_value(self, sender, keyword): self.addToRouter(sender) ret = ["value"] value = self.storage.get(keyword, None) if value is None: return self.rpc_find_node(sender, keyword) ret.extend(value) return ret def rpc_inv(self, sender, *serlialized_invs): self.addToRouter(sender) ret = [] for inv in serlialized_invs: try: i = objects.Inv() i.ParseFromString(inv) if self.storage.getSpecific(i.keyword, i.valueKey) is None: ret.append(inv) except Exception: pass return ret def rpc_values(self, sender, *serialized_values): self.addToRouter(sender) for val in serialized_values[:100]: try: v = objects.Value() v.ParseFromString(val) self.storage[v.keyword] = (v.valueKey, v.serializedData, int(v.ttl)) except Exception: pass return ["True"] def callFindNode(self, nodeToAsk, nodeToFind): d = self.find_node(nodeToAsk, nodeToFind.id) return d.addCallback(self.handleCallResponse, nodeToAsk) def callFindValue(self, nodeToAsk, nodeToFind): d = self.find_value(nodeToAsk, nodeToFind.id) return d.addCallback(self.handleCallResponse, nodeToAsk) def callPing(self, nodeToAsk): d = self.ping(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callStore(self, nodeToAsk, keyword, key, value, ttl): d = self.store(nodeToAsk, keyword, key, value, str(int(round(ttl)))) return d.addCallback(self.handleCallResponse, nodeToAsk) def callDelete(self, nodeToAsk, keyword, key, signature): d = self.delete(nodeToAsk, keyword, key, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callInv(self, nodeToAsk, serlialized_inv_list): d = self.inv(nodeToAsk, *serlialized_inv_list) return d.addCallback(self.handleCallResponse, nodeToAsk) def callValues(self, nodeToAsk, serlialized_values_list): d = self.values(nodeToAsk, *serlialized_values_list) return d.addCallback(self.handleCallResponse, nodeToAsk) def transferKeyValues(self, node): """ Given a new node, send it all the keys/values it should be storing. @param node: A new node that just joined (or that we just found out about). Process: For each key in storage, get k closest nodes. If newnode is closer than the furtherst in that list, and the node for this server is closer than the closest in that list, then store the key/value on the new node (per section 2.5 of the paper) """ def send_values(inv_list): values = [] if inv_list[0]: for requested_inv in inv_list[1]: try: i = objects.Inv() i.ParseFromString(requested_inv) value = self.storage.getSpecific(i.keyword, i.valueKey) if value is not None: v = objects.Value() v.keyword = i.keyword v.valueKey = i.valueKey v.serializedData = value v.ttl = int(round(self.storage.get_ttl(i.keyword, i.valueKey))) values.append(v.SerializeToString()) except Exception: pass if len(values) > 0: self.callValues(node, values) inv = [] for keyword in self.storage.iterkeys(): keyword = keyword[0].decode("hex") keynode = Node(keyword) neighbors = self.router.findNeighbors(keynode, exclude=node) if len(neighbors) > 0: newNodeClose = node.distanceTo(keynode) < neighbors[-1].distanceTo(keynode) thisNodeClosest = self.sourceNode.distanceTo(keynode) < neighbors[0].distanceTo(keynode) if len(neighbors) == 0 \ or (newNodeClose and thisNodeClosest) \ or (thisNodeClosest and len(neighbors) < self.ksize): # pylint: disable=W0612 for k, v in self.storage.iteritems(keyword): i = objects.Inv() i.keyword = keyword i.valueKey = k inv.append(i.SerializeToString()) if len(inv) > 100: random.shuffle(inv) if len(inv) > 0: self.callInv(node, inv[:100]).addCallback(send_values) def handleCallResponse(self, result, node): """ If we get a response, add the node to the routing table. If we get no response, make sure it's removed from the routing table. """ if result[0]: if self.isNewConnection(node) and node.id not in self.recent_transfers: if len(self.recent_transfers) == 10: self.recent_transfers.pop() self.recent_transfers.add(node.id) self.log.debug("call response from new node, transferring key/values") reactor.callLater(1, self.transferKeyValues, node) self.router.addContact(node) else: self.log.debug("no response from %s, removing from router" % node) self.router.removeContact(node) return result def addToRouter(self, node): """ Called by rpc_ functions when a node sends them a request. We add the node to our router and transfer our stored values if they are new and within our neighborhood. """ if self.isNewConnection(node) and node.id not in self.recent_transfers: if len(self.recent_transfers) == 10: self.recent_transfers.pop() self.recent_transfers.add(node.id) self.log.debug("found a new node, transferring key/values") reactor.callLater(1, self.transferKeyValues, node) self.router.addContact(node) def isNewConnection(self, node): if (node.ip, node.port) in self.multiplexer: return self.multiplexer[(node.ip, node.port)].handler.check_new_connection() else: return False def __iter__(self): return iter(self.handled_commands)
class MarketProtocol(RPCProtocol): implements(MessageProcessor) def __init__(self, node, router, signing_key, database): self.router = router self.node = node RPCProtocol.__init__(self, node, router) self.log = Logger(system=self) self.multiplexer = None self.db = database self.signing_key = signing_key self.listeners = [] self.handled_commands = [ GET_CONTRACT, GET_IMAGE, GET_PROFILE, GET_LISTINGS, GET_USER_METADATA, GET_CONTRACT_METADATA, FOLLOW, UNFOLLOW, GET_FOLLOWERS, GET_FOLLOWING, BROADCAST, MESSAGE, ORDER, ORDER_CONFIRMATION, COMPLETE_ORDER, DISPUTE_OPEN, DISPUTE_CLOSE, GET_RATINGS, REFUND ] def connect_multiplexer(self, multiplexer): self.multiplexer = multiplexer def add_listener(self, listener): self.listeners.append(listener) def rpc_get_contract(self, sender, contract_hash): self.log.info("serving contract %s to %s" % (contract_hash.encode('hex'), sender)) self.router.addContact(sender) try: with open(self.db.filemap.get_file(contract_hash.encode("hex")), "r") as filename: contract = filename.read() return [contract] except Exception: self.log.warning("could not find contract %s" % contract_hash.encode('hex')) return None def rpc_get_image(self, sender, image_hash): self.router.addContact(sender) try: if len(image_hash) != 20: self.log.warning("Image hash is not 20 characters %s" % image_hash) raise Exception("Invalid image hash") self.log.info("serving image %s to %s" % (image_hash.encode('hex'), sender)) with open(self.db.filemap.get_file(image_hash.encode("hex")), "rb") as filename: image = filename.read() return [image] except Exception: self.log.warning("could not find image %s" % image_hash[:20].encode('hex')) return None def rpc_get_profile(self, sender): self.log.info("serving profile to %s" % sender) self.router.addContact(sender) try: proto = Profile(self.db).get(True) return [proto, self.signing_key.sign(proto)[:64]] except Exception: self.log.error("unable to load the profile") return None def rpc_get_user_metadata(self, sender): self.log.info("serving user metadata to %s" % sender) self.router.addContact(sender) try: proto = Profile(self.db).get(False) m = Metadata() m.name = proto.name m.handle = proto.handle m.short_description = proto.short_description m.avatar_hash = proto.avatar_hash m.nsfw = proto.nsfw return [ m.SerializeToString(), self.signing_key.sign(m.SerializeToString())[:64] ] except Exception: self.log.error("unable to load profile metadata") return None def rpc_get_listings(self, sender): self.log.info("serving store listings to %s" % sender) self.router.addContact(sender) try: p = Profile(self.db).get() l = Listings() l.ParseFromString(self.db.listings.get_proto()) l.handle = p.handle l.avatar_hash = p.avatar_hash return [ l.SerializeToString(), self.signing_key.sign(l.SerializeToString())[:64] ] except Exception: self.log.warning("could not find any listings in the database") return None def rpc_get_contract_metadata(self, sender, contract_hash): self.log.info("serving metadata for contract %s to %s" % (contract_hash.encode("hex"), sender)) self.router.addContact(sender) try: proto = self.db.listings.get_proto() p = Profile(self.db).get() l = Listings() l.ParseFromString(proto) for listing in l.listing: if listing.contract_hash == contract_hash: listing.avatar_hash = p.avatar_hash listing.handle = p.handle ser = listing.SerializeToString() return [ser, self.signing_key.sign(ser)[:64]] except Exception: self.log.warning("could not find metadata for contract %s" % contract_hash.encode("hex")) return None def rpc_follow(self, sender, proto, signature): self.log.info("received follow request from %s" % sender) self.router.addContact(sender) try: verify_key = nacl.signing.VerifyKey(sender.pubkey) verify_key.verify(proto, signature) f = Followers.Follower() f.ParseFromString(proto) if f.guid != sender.id: raise Exception('GUID does not match sending node') if f.following != self.node.id: raise Exception('Following wrong node') f.signature = signature self.db.follow.set_follower(f) proto = Profile(self.db).get(False) m = Metadata() m.name = proto.name m.handle = proto.handle m.avatar_hash = proto.avatar_hash m.short_description = proto.short_description m.nsfw = proto.nsfw for listener in self.listeners: try: verifyObject(NotificationListener, listener) listener.notify(sender.id, f.metadata.handle, "follow", "", "", f.metadata.avatar_hash) except DoesNotImplement: pass return [ "True", m.SerializeToString(), self.signing_key.sign(m.SerializeToString())[:64] ] except Exception: self.log.warning("failed to validate follower") return ["False"] def rpc_unfollow(self, sender, signature): self.log.info("received unfollow request from %s" % sender) self.router.addContact(sender) try: verify_key = nacl.signing.VerifyKey(sender.pubkey) verify_key.verify("unfollow:" + self.node.id, signature) f = self.db.follow f.delete_follower(sender.id) return ["True"] except Exception: self.log.warning( "failed to validate signature on unfollow request") return ["False"] def rpc_get_followers(self, sender): self.log.info("serving followers list to %s" % sender) self.router.addContact(sender) ser = self.db.follow.get_followers() if ser is None: return None else: return [ser, self.signing_key.sign(ser)[:64]] def rpc_get_following(self, sender): self.log.info("serving following list to %s" % sender) self.router.addContact(sender) ser = self.db.follow.get_following() if ser is None: return None else: return [ser, self.signing_key.sign(ser)[:64]] def rpc_broadcast(self, sender, message, signature): if len(message) <= 140 and self.db.follow.is_following(sender.id): try: verify_key = nacl.signing.VerifyKey(sender.pubkey) verify_key.verify(message, signature) except Exception: self.log.warning("received invalid broadcast from %s" % sender) return ["False"] self.log.info("received a broadcast from %s" % sender) self.router.addContact(sender) for listener in self.listeners: try: verifyObject(BroadcastListener, listener) listener.notify(sender.id, message) except DoesNotImplement: pass return ["True"] else: return ["False"] def rpc_message(self, sender, pubkey, encrypted): try: box = Box(self.signing_key.to_curve25519_private_key(), PublicKey(pubkey)) plaintext = box.decrypt(encrypted) p = PlaintextMessage() p.ParseFromString(plaintext) signature = p.signature p.ClearField("signature") verify_key = nacl.signing.VerifyKey(p.pubkey) verify_key.verify(p.SerializeToString(), signature) h = nacl.hash.sha512(p.pubkey) pow_hash = h[40:] if int(pow_hash[:6], 16) >= 50 or p.sender_guid.encode( "hex") != h[:40] or p.sender_guid != sender.id: raise Exception('Invalid guid') self.log.info("received a message from %s" % sender) self.router.addContact(sender) for listener in self.listeners: try: verifyObject(MessageListener, listener) listener.notify(p, signature) except DoesNotImplement: pass return ["True"] except Exception: self.log.warning("received invalid message from %s" % sender) return ["False"] def rpc_order(self, sender, pubkey, encrypted): try: box = Box(self.signing_key.to_curve25519_private_key(), PublicKey(pubkey)) order = box.decrypt(encrypted) c = Contract(self.db, contract=json.loads(order, object_pairs_hook=OrderedDict), testnet=self.multiplexer.testnet) if c.verify(sender.pubkey): self.router.addContact(sender) self.log.info( "received an order from %s, waiting for payment..." % sender) payment_address = c.contract["buyer_order"]["order"][ "payment"]["address"] chaincode = c.contract["buyer_order"]["order"]["payment"][ "chaincode"] masterkey_b = c.contract["buyer_order"]["order"]["id"][ "pubkeys"]["bitcoin"] buyer_key = derive_childkey(masterkey_b, chaincode) amount = c.contract["buyer_order"]["order"]["payment"][ "amount"] listing_hash = c.contract["vendor_offer"]["listing"][ "contract_id"] signature = self.signing_key.sign( str(payment_address) + str(amount) + str(listing_hash) + str(buyer_key))[:64] c.await_funding(self.get_notification_listener(), self.multiplexer.blockchain, signature, False) return [signature] else: self.log.warning("received invalid order from %s" % sender) return ["False"] except Exception: self.log.error("unable to decrypt order from %s" % sender) return ["False"] def rpc_order_confirmation(self, sender, pubkey, encrypted): try: box = Box(self.signing_key.to_curve25519_private_key(), PublicKey(pubkey)) order = box.decrypt(encrypted) c = Contract(self.db, contract=json.loads(order, object_pairs_hook=OrderedDict), testnet=self.multiplexer.testnet) contract_id = c.accept_order_confirmation( self.get_notification_listener()) if contract_id: self.router.addContact(sender) self.log.info("received confirmation for order %s" % contract_id) return ["True"] else: self.log.warning( "received invalid order confirmation from %s" % sender) return ["False"] except Exception: self.log.error("unable to decrypt order confirmation from %s" % sender) return ["False"] def rpc_complete_order(self, sender, pubkey, encrypted): try: box = Box(self.signing_key.to_curve25519_private_key(), PublicKey(pubkey)) order = box.decrypt(encrypted) c = Contract(self.db, contract=json.loads(order, object_pairs_hook=OrderedDict), testnet=self.multiplexer.testnet) contract_id = c.accept_receipt(self.get_notification_listener(), self.multiplexer.blockchain) self.router.addContact(sender) self.log.info("received receipt for order %s" % contract_id) return ["True"] except Exception: import traceback traceback.print_exc() self.log.error("unable to parse receipt from %s" % sender) return ["False"] def rpc_dispute_open(self, sender, pubkey, encrypted): try: box = Box(self.signing_key.to_curve25519_private_key(), PublicKey(pubkey)) order = box.decrypt(encrypted) contract = json.loads(order, object_pairs_hook=OrderedDict) process_dispute(contract, self.db, self.get_message_listener(), self.get_notification_listener(), self.multiplexer.testnet) self.router.addContact(sender) self.log.info("Contract dispute opened by %s" % sender) return ["True"] except Exception: self.log.error("unable to parse disputed contract from %s" % sender) return ["False"] def rpc_dispute_close(self, sender, pubkey, encrypted): try: box = Box(self.signing_key.to_curve25519_private_key(), PublicKey(pubkey)) res = box.decrypt(encrypted) resolution_json = json.loads(res, object_pairs_hook=OrderedDict) close_dispute(resolution_json, self.db, self.get_message_listener(), self.get_notification_listener(), self.multiplexer.testnet) self.router.addContact(sender) self.log.info("Contract dispute closed by %s" % sender) return ["True"] except Exception: self.log.error("unable to parse disputed close message from %s" % sender) return ["False"] def rpc_get_ratings(self, sender, listing_hash=None): a = "ALL" if listing_hash is None else listing_hash.encode("hex") self.log.info("serving ratings for contract %s to %s" % (a, sender)) self.router.addContact(sender) try: ratings = [] if listing_hash: for rating in self.db.ratings.get_listing_ratings( listing_hash.encode("hex")): ratings.append( json.loads(rating[0], object_pairs_hook=OrderedDict)) else: for rating in self.db.ratings.get_all_ratings(): ratings.append( json.loads(rating[0], object_pairs_hook=OrderedDict)) ret = json.dumps(ratings).encode("zlib") return [str(ret), self.signing_key.sign(ret)[:64]] except Exception: self.log.warning("could not load ratings for contract %s" % a) return None def rpc_refund(self, sender, pubkey, encrypted): try: box = Box(self.signing_key.to_curve25519_private_key(), PublicKey(pubkey)) refund = box.decrypt(encrypted) refund_json = json.loads(refund, object_pairs_hook=OrderedDict) c = Contract(self.db, hash_value=unhexlify( refund_json["refund"]["order_id"]), testnet=self.multiplexer.testnet) c.process_refund(refund_json, self.multiplexer.blockchain, self.get_notification_listener()) self.router.addContact(sender) self.log.info("order %s refunded by vendor" % refund_json["refund"]["order_id"]) return ["True"] except Exception: self.log.error("unable to parse refund message from %s" % sender) return ["False"] def callGetContract(self, nodeToAsk, contract_hash): d = self.get_contract(nodeToAsk, contract_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetImage(self, nodeToAsk, image_hash): d = self.get_image(nodeToAsk, image_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetProfile(self, nodeToAsk): d = self.get_profile(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetUserMetadata(self, nodeToAsk): d = self.get_user_metadata(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetListings(self, nodeToAsk): d = self.get_listings(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetContractMetadata(self, nodeToAsk, contract_hash): d = self.get_contract_metadata(nodeToAsk, contract_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callFollow(self, nodeToAsk, proto, signature): d = self.follow(nodeToAsk, proto, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callUnfollow(self, nodeToAsk, signature): d = self.unfollow(nodeToAsk, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetFollowers(self, nodeToAsk): d = self.get_followers(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetFollowing(self, nodeToAsk): d = self.get_following(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callBroadcast(self, nodeToAsk, message, signature): d = self.broadcast(nodeToAsk, message, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callMessage(self, nodeToAsk, ehemeral_pubkey, ciphertext): d = self.message(nodeToAsk, ehemeral_pubkey, ciphertext) return d.addCallback(self.handleCallResponse, nodeToAsk) def callOrder(self, nodeToAsk, ephem_pubkey, encrypted_contract): d = self.order(nodeToAsk, ephem_pubkey, encrypted_contract) return d.addCallback(self.handleCallResponse, nodeToAsk) def callOrderConfirmation(self, nodeToAsk, ephem_pubkey, encrypted_contract): d = self.order_confirmation(nodeToAsk, ephem_pubkey, encrypted_contract) return d.addCallback(self.handleCallResponse, nodeToAsk) def callCompleteOrder(self, nodeToAsk, ephem_pubkey, encrypted_contract): d = self.complete_order(nodeToAsk, ephem_pubkey, encrypted_contract) return d.addCallback(self.handleCallResponse, nodeToAsk) def callDisputeOpen(self, nodeToAsk, ephem_pubkey, encrypted_contract): d = self.dispute_open(nodeToAsk, ephem_pubkey, encrypted_contract) return d.addCallback(self.handleCallResponse, nodeToAsk) def callDisputeClose(self, nodeToAsk, ephem_pubkey, encrypted_contract): d = self.dispute_close(nodeToAsk, ephem_pubkey, encrypted_contract) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetRatings(self, nodeToAsk, listing_hash=None): if listing_hash is None: d = self.get_ratings(nodeToAsk) else: d = self.get_ratings(nodeToAsk, listing_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callRefund(self, nodeToAsk, order_id, refund): d = self.refund(nodeToAsk, order_id, refund) return d.addCallback(self.handleCallResponse, nodeToAsk) def handleCallResponse(self, result, node): """ If we get a response, add the node to the routing table. If we get no response, make sure it's removed from the routing table. """ if result[0]: self.router.addContact(node) else: self.log.debug("no response from %s, removing from router" % node) self.router.removeContact(node) return result def get_notification_listener(self): for listener in self.listeners: try: verifyObject(NotificationListener, listener) return listener except DoesNotImplement: pass def get_message_listener(self): for listener in self.listeners: try: verifyObject(MessageListener, listener) return listener except DoesNotImplement: pass def __iter__(self): return iter(self.handled_commands)
class RPCProtocol: """ This is an abstract class for processing and sending rpc messages. A class that implements the `MessageProcessor` interface probably should extend this as it does most of the work of keeping track of messages. """ __metaclass__ = abc.ABCMeta def __init__(self, sourceNode, router, waitTimeout=2.5): """ Args: proto: A protobuf `Node` object containing info about this node. router: A `RoutingTable` object from dht.routing. Implies a `network.Server` object must be started first. waitTimeout: Consider it a connetion failure if no response within this time window. noisy: Whether or not to log the output for this class. testnet: The network parameters to use. """ self.sourceNode = sourceNode self.router = router self._waitTimeout = waitTimeout self._outstanding = {} self.log = Logger(system=self) def receive_message(self, datagram, connection, ban_score): m = Message() try: m.ParseFromString(datagram) sender = node.Node(m.sender.guid, m.sender.ip, m.sender.port, m.sender.signedPublicKey, m.sender.vendor) except Exception: # If message isn't formatted property then ignore self.log.warning("received unknown message from %s, ignoring" % str(connection.dest_addr)) return False if m.testnet != self.multiplexer.testnet: self.log.warning("received message from %s with incorrect network parameters." % str(connection.dest_addr)) connection.shutdown() return False if m.protoVer < PROTOCOL_VERSION: self.log.warning("received message from %s with incompatible protocol version." % str(connection.dest_addr)) connection.shutdown() return False # Check that the GUID is valid. If not, ignore if self.router.isNewNode(sender): try: pubkey = m.sender.signedPublicKey[len(m.sender.signedPublicKey) - 32:] verify_key = nacl.signing.VerifyKey(pubkey) verify_key.verify(m.sender.signedPublicKey) h = nacl.hash.sha512(m.sender.signedPublicKey) pow_hash = h[64:128] if int(pow_hash[:6], 16) >= 50 or hexlify(m.sender.guid) != h[:40]: raise Exception('Invalid GUID') except Exception: self.log.warning("received message from sender with invalid GUID, ignoring") connection.shutdown() return False if m.sender.vendor: self.db.VendorStore().save_vendor(m.sender.guid.encode("hex"), m.sender.ip, m.sender.port, m.sender.signedPublicKey) msgID = m.messageID if m.command == NOT_FOUND: data = None else: data = tuple(m.arguments) if msgID in self._outstanding: self._acceptResponse(msgID, data, sender) elif m.command != NOT_FOUND: #ban_score.process_message(m) self._acceptRequest(msgID, str(Command.Name(m.command)).lower(), data, sender, connection) def _acceptResponse(self, msgID, data, sender): if data is not None: msgargs = (b64encode(msgID), sender) self.log.debug("received response for message id %s from %s" % msgargs) else: self.log.warning("received 404 error response from %s" % sender) d = self._outstanding[msgID][0] if self._outstanding[msgID][2].active(): self._outstanding[msgID][2].cancel() d.callback((True, data)) del self._outstanding[msgID] def _acceptRequest(self, msgID, funcname, args, sender, connection): self.log.debug("received request from %s, command %s" % (sender, funcname.upper())) f = getattr(self, "rpc_%s" % funcname, None) if f is None or not callable(f): msgargs = (self.__class__.__name__, funcname) self.log.error("%s has no callable method rpc_%s; ignoring request" % msgargs) return False if funcname == "hole_punch": f(sender, *args) else: d = defer.maybeDeferred(f, sender, *args) d.addCallback(self._sendResponse, funcname, msgID, sender, connection) d.addErrback(self._sendResponse, "bad_request", msgID, sender, connection) def _sendResponse(self, response, funcname, msgID, sender, connection): self.log.debug("sending response for msg id %s to %s" % (b64encode(msgID), sender)) m = Message() m.messageID = msgID m.sender.MergeFrom(self.sourceNode.getProto()) m.protoVer = PROTOCOL_VERSION m.testnet = self.multiplexer.testnet if response is None: m.command = NOT_FOUND else: m.command = Command.Value(funcname.upper()) if not isinstance(response, list): response = [response] for arg in response: m.arguments.append(str(arg)) connection.send_message(m.SerializeToString()) def timeout(self, address): """ This timeout is called by the txrudp connection handler. We will run through the outstanding messages and callback false on any waiting on this IP address. """ for msgID, val in self._outstanding.items(): if address == val[1]: val[0].callback((False, None)) del self._outstanding[msgID] try: node_to_remove = self.multiplexer[address].handler.node if node_to_remove is not None: self.router.removeContact(node_to_remove) self.multiplexer[address].shutdown() except Exception: pass def rpc_hole_punch(self, sender, ip, port, relay="False"): """ A method for handling an incoming HOLE_PUNCH message. Relay the message to the correct node if it's not for us. Otherwise send a datagram to allow the other node to punch through our NAT. """ if relay == "True": self.hole_punch((ip, int(port)), sender.ip, sender.port) else: self.log.debug("punching through NAT for %s:%s" % (ip, port)) # pylint: disable=W0612 for i in range(20): self.multiplexer.send_datagram("", (ip, int(port))) def __getattr__(self, name): if name.startswith("_") or name.startswith("rpc_"): return object.__getattr__(self, name) try: return object.__getattr__(self, name) except AttributeError: pass def func(address, *args): msgID = sha1(str(random.getrandbits(255))).digest() d = defer.Deferred() if name != "hole_punch": seed = SEED_NODE_TESTNET if self.multiplexer.testnet else SEED_NODE if address in self.multiplexer and self.multiplexer[address].state == State.CONNECTED: timeout = timeout = reactor.callLater(self._waitTimeout, self.timeout, address) else: timeout = reactor.callLater(self._waitTimeout, self.hole_punch, seed, address[0], address[1], "True", msgID) self._outstanding[msgID] = [d, address, timeout] self.log.debug("calling remote function %s on %s (msgid %s)" % (name, address, b64encode(msgID))) elif args[3] in self._outstanding: prev_msgID = args[3] args = args[:3] deferred, addr, hp = self._outstanding[prev_msgID] # pylint: disable=W0612 timeout = reactor.callLater(3, self.timeout, addr) self._outstanding[prev_msgID] = [deferred, addr, timeout] self.log.debug("sending hole punch message to %s" % args[0] + ":" + str(args[1])) m = Message() m.messageID = msgID m.sender.MergeFrom(self.sourceNode.getProto()) m.command = Command.Value(name.upper()) m.protoVer = PROTOCOL_VERSION for arg in args: m.arguments.append(str(arg)) m.testnet = self.multiplexer.testnet data = m.SerializeToString() self.multiplexer.send_message(data, address) return d return func
class MarketProtocol(RPCProtocol): implements(MessageProcessor) def __init__(self, node, router, signing_key, database): self.router = router self.node = node RPCProtocol.__init__(self, node, router) self.log = Logger(system=self) self.multiplexer = None self.db = database self.signing_key = signing_key self.listeners = [] self.handled_commands = [GET_CONTRACT, GET_IMAGE, GET_PROFILE, GET_LISTINGS, GET_USER_METADATA, GET_CONTRACT_METADATA, FOLLOW, UNFOLLOW, GET_FOLLOWERS, GET_FOLLOWING, BROADCAST, MESSAGE, ORDER, ORDER_CONFIRMATION, COMPLETE_ORDER, DISPUTE_OPEN, DISPUTE_CLOSE] def connect_multiplexer(self, multiplexer): self.multiplexer = multiplexer def add_listener(self, listener): self.listeners.append(listener) def rpc_get_contract(self, sender, contract_hash): self.log.info("serving contract %s to %s" % (contract_hash.encode('hex'), sender)) self.router.addContact(sender) try: with open(self.db.HashMap().get_file(contract_hash.encode("hex")), "r") as filename: contract = filename.read() return [contract] except Exception: self.log.warning("could not find contract %s" % contract_hash.encode('hex')) return None def rpc_get_image(self, sender, image_hash): self.router.addContact(sender) try: if len(image_hash) != 20: raise Exception("Invalid image hash") self.log.info("serving image %s to %s" % (image_hash.encode('hex'), sender)) with open(self.db.HashMap().get_file(image_hash.encode("hex")), "rb") as filename: image = filename.read() return [image] except Exception: self.log.warning("could not find image %s" % image_hash[:20].encode('hex')) return None def rpc_get_profile(self, sender): self.log.info("serving profile to %s" % sender) self.router.addContact(sender) try: proto = Profile(self.db).get(True) return [proto, self.signing_key.sign(proto)[:64]] except Exception: self.log.error("unable to load the profile") return None def rpc_get_user_metadata(self, sender): self.log.info("serving user metadata to %s" % sender) self.router.addContact(sender) try: proto = Profile(self.db).get(False) m = Metadata() m.name = proto.name m.handle = proto.handle m.short_description = proto.short_description m.avatar_hash = proto.avatar_hash m.nsfw = proto.nsfw return [m.SerializeToString(), self.signing_key.sign(m.SerializeToString())[:64]] except Exception: self.log.error("unable to load profile metadata") return None def rpc_get_listings(self, sender): self.log.info("serving store listings to %s" % sender) self.router.addContact(sender) try: p = Profile(self.db).get() l = Listings() l.ParseFromString(self.db.ListingsStore().get_proto()) l.handle = p.handle l.avatar_hash = p.avatar_hash return [l.SerializeToString(), self.signing_key.sign(l.SerializeToString())[:64]] except Exception: self.log.warning("could not find any listings in the database") return None def rpc_get_contract_metadata(self, sender, contract_hash): self.log.info("serving metadata for contract %s to %s" % (contract_hash.encode("hex"), sender)) self.router.addContact(sender) try: proto = self.db.ListingsStore().get_proto() p = Profile(self.db).get() l = Listings() l.ParseFromString(proto) for listing in l.listing: if listing.contract_hash == contract_hash: listing.avatar_hash = p.avatar_hash listing.handle = p.handle ser = listing.SerializeToString() return [ser, self.signing_key.sign(ser)[:64]] except Exception: self.log.warning("could not find metadata for contract %s" % contract_hash.encode("hex")) return None def rpc_follow(self, sender, proto, signature): self.log.info("received follow request from %s" % sender) self.router.addContact(sender) try: verify_key = nacl.signing.VerifyKey(sender.signed_pubkey[64:]) verify_key.verify(proto, signature) f = Followers.Follower() f.ParseFromString(proto) if f.guid != sender.id: raise Exception('GUID does not match sending node') if f.following != self.node.id: raise Exception('Following wrong node') f.signature = signature self.db.FollowData().set_follower(f) proto = Profile(self.db).get(False) m = Metadata() m.name = proto.name m.handle = proto.handle m.avatar_hash = proto.avatar_hash m.short_description = proto.short_description m.nsfw = proto.nsfw for listener in self.listeners: try: verifyObject(NotificationListener, listener) listener.notify(sender.id, f.metadata.handle, "follow", "", "", f.metadata.avatar_hash) except DoesNotImplement: pass return ["True", m.SerializeToString(), self.signing_key.sign(m.SerializeToString())[:64]] except Exception: self.log.warning("failed to validate follower") return ["False"] def rpc_unfollow(self, sender, signature): self.log.info("received unfollow request from %s" % sender) self.router.addContact(sender) try: verify_key = nacl.signing.VerifyKey(sender.signed_pubkey[64:]) verify_key.verify("unfollow:" + self.node.id, signature) f = self.db.FollowData() f.delete_follower(sender.id) return ["True"] except Exception: self.log.warning("failed to validate signature on unfollow request") return ["False"] def rpc_get_followers(self, sender): self.log.info("serving followers list to %s" % sender) self.router.addContact(sender) ser = self.db.FollowData().get_followers() if ser is None: return None else: return [ser, self.signing_key.sign(ser)[:64]] def rpc_get_following(self, sender): self.log.info("serving following list to %s" % sender) self.router.addContact(sender) ser = self.db.FollowData().get_following() if ser is None: return None else: return [ser, self.signing_key.sign(ser)[:64]] def rpc_broadcast(self, sender, message, signature): if len(message) <= 140 and self.db.FollowData().is_following(sender.id): try: verify_key = nacl.signing.VerifyKey(sender.signed_pubkey[64:]) verify_key.verify(message, signature) except Exception: self.log.warning("received invalid broadcast from %s" % sender) return ["False"] self.log.info("received a broadcast from %s" % sender) self.router.addContact(sender) for listener in self.listeners: try: verifyObject(BroadcastListener, listener) listener.notify(sender.id, message) except DoesNotImplement: pass return ["True"] else: return ["False"] def rpc_message(self, sender, pubkey, encrypted): try: box = Box(PrivateKey(self.signing_key.encode(nacl.encoding.RawEncoder)), PublicKey(pubkey)) plaintext = box.decrypt(encrypted) p = PlaintextMessage() p.ParseFromString(plaintext) signature = p.signature p.ClearField("signature") verify_key = nacl.signing.VerifyKey(p.signed_pubkey[64:]) verify_key.verify(p.SerializeToString(), signature) h = nacl.hash.sha512(p.signed_pubkey) pow_hash = h[64:128] if int(pow_hash[:6], 16) >= 50 or p.sender_guid.encode("hex") != h[:40] or p.sender_guid != sender.id: raise Exception('Invalid guid') self.log.info("received a message from %s" % sender) self.router.addContact(sender) for listener in self.listeners: try: verifyObject(MessageListener, listener) listener.notify(p, signature) except DoesNotImplement: pass return ["True"] except Exception: self.log.warning("received invalid message from %s" % sender) return ["False"] def rpc_order(self, sender, pubkey, encrypted): try: box = Box(PrivateKey(self.signing_key.encode(nacl.encoding.RawEncoder)), PublicKey(pubkey)) order = box.decrypt(encrypted) c = Contract(self.db, contract=json.loads(order, object_pairs_hook=OrderedDict), testnet=self.multiplexer.testnet) if c.verify(sender.signed_pubkey[64:]): self.router.addContact(sender) self.log.info("received an order from %s, waiting for payment..." % sender) payment_address = c.contract["buyer_order"]["order"]["payment"]["address"] chaincode = c.contract["buyer_order"]["order"]["payment"]["chaincode"] masterkey_b = c.contract["buyer_order"]["order"]["id"]["pubkeys"]["bitcoin"] buyer_key = derive_childkey(masterkey_b, chaincode) amount = c.contract["buyer_order"]["order"]["payment"]["amount"] listing_hash = c.contract["buyer_order"]["order"]["ref_hash"] signature = self.signing_key.sign( str(payment_address) + str(amount) + str(listing_hash) + str(buyer_key))[:64] c.await_funding(self.get_notification_listener(), self.multiplexer.blockchain, signature, False) return [signature] else: self.log.warning("received invalid order from %s" % sender) return ["False"] except Exception: self.log.error("unable to decrypt order from %s" % sender) return ["False"] def rpc_order_confirmation(self, sender, pubkey, encrypted): try: box = Box(PrivateKey(self.signing_key.encode(nacl.encoding.RawEncoder)), PublicKey(pubkey)) order = box.decrypt(encrypted) c = Contract(self.db, contract=json.loads(order, object_pairs_hook=OrderedDict), testnet=self.multiplexer.testnet) contract_id = c.accept_order_confirmation(self.get_notification_listener()) if contract_id: self.router.addContact(sender) self.log.info("received confirmation for order %s" % contract_id) return ["True"] else: self.log.warning("received invalid order confirmation from %s" % sender) return ["False"] except Exception: self.log.error("unable to decrypt order confirmation from %s" % sender) return ["False"] def rpc_complete_order(self, sender, pubkey, encrypted): try: box = Box(PrivateKey(self.signing_key.encode(nacl.encoding.RawEncoder)), PublicKey(pubkey)) order = box.decrypt(encrypted) c = Contract(self.db, contract=json.loads(order, object_pairs_hook=OrderedDict), testnet=self.multiplexer.testnet) contract_id = c.accept_receipt(self.get_notification_listener(), self.multiplexer.blockchain) self.router.addContact(sender) self.log.info("received receipt for order %s" % contract_id) return ["True"] except Exception: self.log.error("unable to parse receipt from %s" % sender) return ["False"] def rpc_dispute_open(self, sender, pubkey, encrypted): try: box = Box(PrivateKey(self.signing_key.encode(nacl.encoding.RawEncoder)), PublicKey(pubkey)) order = box.decrypt(encrypted) contract = json.loads(order, object_pairs_hook=OrderedDict) process_dispute(contract, self.db, self.get_message_listener(), self.get_notification_listener(), self.multiplexer.testnet) self.router.addContact(sender) self.log.info("Contract dispute opened by %s" % sender) return ["True"] except Exception: self.log.error("unable to parse disputed contract from %s" % sender) return ["False"] def rpc_dispute_close(self, sender, pubkey, encrypted): try: box = Box(PrivateKey(self.signing_key.encode(nacl.encoding.RawEncoder)), PublicKey(pubkey)) order = box.decrypt(encrypted) contract = json.loads(order, object_pairs_hook=OrderedDict) close_dispute(contract, self.db, self.get_message_listener(), self.get_notification_listener(), self.multiplexer.testnet) self.router.addContact(sender) self.log.info("Contract dispute closed by %s" % sender) return ["True"] except Exception: self.log.error("unable to parse disputed close message from %s" % sender) return ["False"] def callGetContract(self, nodeToAsk, contract_hash): d = self.get_contract(nodeToAsk, contract_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetImage(self, nodeToAsk, image_hash): d = self.get_image(nodeToAsk, image_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetProfile(self, nodeToAsk): d = self.get_profile(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetUserMetadata(self, nodeToAsk): d = self.get_user_metadata(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetListings(self, nodeToAsk): d = self.get_listings(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetContractMetadata(self, nodeToAsk, contract_hash): d = self.get_contract_metadata(nodeToAsk, contract_hash) return d.addCallback(self.handleCallResponse, nodeToAsk) def callFollow(self, nodeToAsk, proto, signature): d = self.follow(nodeToAsk, proto, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callUnfollow(self, nodeToAsk, signature): d = self.unfollow(nodeToAsk, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetFollowers(self, nodeToAsk): d = self.get_followers(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callGetFollowing(self, nodeToAsk): d = self.get_following(nodeToAsk) return d.addCallback(self.handleCallResponse, nodeToAsk) def callBroadcast(self, nodeToAsk, message, signature): d = self.broadcast(nodeToAsk, message, signature) return d.addCallback(self.handleCallResponse, nodeToAsk) def callMessage(self, nodeToAsk, ehemeral_pubkey, ciphertext): d = self.message(nodeToAsk, ehemeral_pubkey, ciphertext) return d.addCallback(self.handleCallResponse, nodeToAsk) def callOrder(self, nodeToAsk, ephem_pubkey, encrypted_contract): d = self.order(nodeToAsk, ephem_pubkey, encrypted_contract) return d.addCallback(self.handleCallResponse, nodeToAsk) def callOrderConfirmation(self, nodeToAsk, ephem_pubkey, encrypted_contract): d = self.order_confirmation(nodeToAsk, ephem_pubkey, encrypted_contract) return d.addCallback(self.handleCallResponse, nodeToAsk) def callCompleteOrder(self, nodeToAsk, ephem_pubkey, encrypted_contract): d = self.complete_order(nodeToAsk, ephem_pubkey, encrypted_contract) return d.addCallback(self.handleCallResponse, nodeToAsk) def callDisputeOpen(self, nodeToAsk, ephem_pubkey, encrypted_contract): d = self.dispute_open(nodeToAsk, ephem_pubkey, encrypted_contract) return d.addCallback(self.handleCallResponse, nodeToAsk) def callDisputeClose(self, nodeToAsk, ephem_pubkey, encrypted_contract): d = self.dispute_open(nodeToAsk, ephem_pubkey, encrypted_contract) return d.addCallback(self.handleCallResponse, nodeToAsk) def handleCallResponse(self, result, node): """ If we get a response, add the node to the routing table. If we get no response, make sure it's removed from the routing table. """ if result[0]: self.router.addContact(node) else: self.log.debug("no response from %s, removing from router" % node) self.router.removeContact(node) return result def get_notification_listener(self): for listener in self.listeners: try: verifyObject(NotificationListener, listener) return listener except DoesNotImplement: pass def get_message_listener(self): for listener in self.listeners: try: verifyObject(MessageListener, listener) return listener except DoesNotImplement: pass def __iter__(self): return iter(self.handled_commands)
class Server(object): """ High level view of a node instance. This is the object that should be created to start listening as an active node on the network. """ def __init__(self, node, db, ksize=20, alpha=3, storage=None): """ Create a server instance. This will start listening on the given port. Args: node: The node instance for this peer. It must contain (at minimum) an ID, public key, ip address, and port. ksize (int): The k parameter from the paper alpha (int): The alpha parameter from the paper storage: An instance that implements :interface:`~dht.storage.IStorage` """ self.ksize = ksize self.alpha = alpha self.log = Logger(system=self) self.storage = storage or ForgetfulStorage() self.node = node self.protocol = KademliaProtocol(self.node, self.storage, ksize, db) self.refreshLoop = LoopingCall(self.refreshTable).start(3600) def listen(self, port): """ Start listening on the given port. This is the same as calling:: reactor.listenUDP(port, server.protocol) """ return reactor.listenUDP(port, self.protocol) def refreshTable(self): """ Refresh buckets that haven't had any lookups in the last hour (per section 2.3 of the paper). """ ds = [] for rid in self.protocol.getRefreshIDs(): node = Node(rid) nearest = self.protocol.router.findNeighbors(node, self.alpha) spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) ds.append(spider.find()) def republishKeys(_): ds = [] # Republish keys older than one hour for keyword in self.storage.iterkeys(): for k, v in self.storage.iteritems(keyword): if self.storage.get_ttl(keyword, k) < 601200: ds.append(self.set(keyword, k, v)) return defer.gatherResults(ds).addCallback(republishKeys) def querySeed(self, seed, pubkey): """ Query an HTTP seed and return a `list` if (ip, port) `tuple` pairs. Args: seed: A `string` consisting of "ip:port" or "hostname:port" pubkey: The hex encoded public key to verify the signature on the response """ try: self.log.info("querying %s for peers" % seed) nodes = [] c = httplib.HTTPConnection(seed) c.request("GET", "/") response = c.getresponse() self.log.debug("Http response from %s: %s, %s" % (seed, response.status, response.reason)) data = response.read() reread_data = data.decode("zlib") proto = peers.PeerSeeds() proto.ParseFromString(reread_data) for peer in proto.peer_data: p = peers.PeerData() p.ParseFromString(peer) tup = (str(p.ip_address), p.port) nodes.append(tup) verify_key = nacl.signing.VerifyKey(pubkey, encoder=nacl.encoding.HexEncoder) verify_key.verify("".join(proto.peer_data), proto.signature) self.log.info("%s returned %s addresses" % (seed, len(nodes))) return nodes except Exception, e: self.log.error("failed to query seed: %s" % str(e))