def run(self): while True: size = queue.qsize() print size try: announce = queue.get() ''' announce = queue.pop() if announce is None: time.sleep(1) continue announce = json.loads(announce) announce[1] = announce[1].decode('hex') announce[0] = (announce[0][0], announce[0][1]) print announce[0] ''' r = simMetadata.download_metadata(announce[0], announce[1]) #print r if r is None: continue name = r['name'] if isinstance(name, str): try: code = chardet.detect(name) name = name.decode(code['encoding']) except UnicodeDecodeError: try: name = name.decode('utf8') except: continue except TypeError: continue try: lang = detect(name) if lang not in ('ja', 'ko', 'zh-cn', 'zh-tw', 'zh-hk', 'zh-sg'):#'en', 'en-us', ): continue except lang_detect_exception.LangDetectException: lang = u'' name = name.encode('utf8') lang = lang.encode('utf8').strip() else: print '***' file_list = r.get('files', []) l = len(file_list) ret = [] for i in xrange(0, l): if 'path.utf-8' in file_list[i]: f = file_list[i]['path.utf-8'][0] else: f = file_list[i]['path'][0] try: code = chardet.detect(f) f = f.decode(code['encoding']) except UnicodeDecodeError: f = u'' #file_list[i]['path'][0] = f.encode('utf8') ret.append({'path': f.encode('utf-8'), 'length': file_list[i]['length']}) #if not check_format(name, file_list): # print 'not video' # continue json.dumps({'name':name}) json.dumps({'files':ret}) json.dumps({'lang':lang}) hashcode = announce[1].encode('hex') print hashcode json.dumps({'hashcode':hashcode}) r = base64.b64encode(bencode(r)) json.dumps({'raw':r}) print "%s magnet:?xt=urn:btih:%s" %(name, hashcode) try: body = json.dumps({ 'raw': r, 'hashcode': hashcode, 'meta': { 'name': name, 'files': ret, 'source': 'test-node', 'lang': lang, } }) message_queue.put(body) except UnicodeError: print 'ex-lang=', lang, type(lang) print 'ex-name=', name, type(name) print 'ex-files', file_list except Exception, e: import traceback print traceback.format_exc() time.sleep(1) except Queue.Empty: print 'Queue Empty' time.sleep(1)
def handle_query(self, message): trans_id = message["t"] query_type = message["q"] args = message["a"] node_id = args["id"] client_host, client_port = self.client_address # logger.debug("Query message %s from %s:%d, id:%r" % (query_type, client_host, client_port, node_id.encode("hex"))) # Do we know already about this node? node = self.server.dht.rt.node_by_id(node_id) if not node: node = Node(client_host, client_port, node_id) # logger.debug("We don`t know about %r, add it as new" % (node)) self.server.dht.rt.update_node(node_id, node) else: pass # logger.debug("We already know about: %r" % (node)) node.update_access() if query_type == "ping": # logger.debug("handle query ping") node.pong(socket=self.server.socket, trans_id = trans_id, sender_id=self.server.dht.node._id, lock=self.server.send_lock) elif query_type == "find_node": # logger.debug("handle query find_node") target = args["target"] found_nodes = encode_nodes(self.server.dht.rt.get_close_nodes(target, 8)) node.found_node(found_nodes, socket=self.server.socket, trans_id = trans_id, sender_id=self.server.dht.node._id, lock=self.server.send_lock) elif query_type == "get_peers": # logger.debug("handle query get_peers") #node.pong(socket=self.server.socket, trans_id = trans_id, sender_id=self.server.dht.node._id, lock=self.server.send_lock) info_hash = args["info_hash"] token = create_token(self.server.dht.key,info_hash,node_id) nodes = None values = [] peers = self.server.dht.ht.get_hash_peers(info_hash.encode("hex")) if not peers: nodes = encode_nodes(self.server.dht.rt.get_close_nodes(info_hash, 8)) else: for p in peers: values.append(pack_hostport(p[0],p[1])) node.got_peers(token, values, nodes, socket=self.server.socket, trans_id=trans_id, sender_id=self.server.dht.node._id, lock=self.server.send_lock) # store token for later check if announce_peer received node.add_local_token(info_hash, token) return elif query_type == "announce_peer": logger.debug("handle query announce_peer") node.pong(socket=self.server.socket, trans_id=trans_id, sender_id=self.server.dht.node._id, lock=self.server.send_lock) info_hash = args["info_hash"].encode("hex") self.server.dht.ht.add_peer(info_hash, (node.host, node.port)) logger.debug('download ...........................') simMetadata.download_metadata((node.host, node.port), info_hash) ## check on token (need to store them beforehand when responding to get_peers) # token = args['token'] # info_hash = args['info_hash'] # node_id = args['id'] # if not token: # let's ignore the call # logger.debug('No token received by ' % (node_id.encode('hex'))) # else: # ctoken = node.get_local_token(info_hash) # if ctoken and token == ctoken: # node.pong(socket=self.server.socket, trans_id = trans_id, sender_id=self.server.dht.node._id, lock=self.server.send_lock) # self.server.dht.ht.add_peer(info_hash, (node.host, node.port)) # else: # node.send_protocol_error("Bad token", socket=self.server.socket, trans_id = trans_id, lock=self.server.send_lock) return else: logger.error("Unknown query type: %s" % (query_type))