def dispatch(self, request, *args, **kwargs): with Session() as s: addr = kwargs['node'].replace('+', '/') if addr.startswith(':') or addr.startswith('/'): addr = request.META['REMOTE_ADDR'] + addr response = HttpResponse() thisNode = Node.getThisNode(s, addr).first() otherNode = Node.getOtherNode(s, addr).first() linkedNodeCount = Node.getLinkedNode(s).count() welcome = False if thisNode: welcome = True thisNode.linked = True thisNode.updateTimestamp() s.commit() elif linkedNodeCount < settings.MAX_NODES: welcome = True s.add(Node(host=addr, linked=True)) s.commit() if welcome: response.write('WELCOME') if otherNode: response.write('\n'+str(otherNode.host)) return response
def doPing(msg): with Session() as s: queue = Queue() for node in Node.getLinkedNode(s).all(): queue.put((node.host,)) multiThread(_doPing_worker, queue, maxWorkers=settings.MAX_CONNECTIONS) MessageQueue.enqueue(s, msgtype='join', msg='init')
def getRecent(msg): log.isEnabledFor(logging.INFO) and log.info('getRecent: {}'.format(msg.msg)) with Session() as s: queue = Queue() for node in Node.getLinkedNode(s).all(): queue.put((node.host,)) multiThread(_getRecent_worker, queue, maxWorkers=settings.MAX_CONNECTIONS)
def updateRecord(msg): with Session() as s: thread_id, hex_id, atime, nodeName = msg.msg.split() thread_id = int(thread_id) atime = int(atime) filename = Thread.getFileName(Thread.get(s, id=thread_id).value(Thread.title)) queue = Queue() for host in Node.getLinkedNode(s).values(Node.host): queue.put(( host.host, filename, atime, hex_id, thread_id, nodeName )) log.isEnabledFor(logging.INFO) and log.info('updateRecord: Run {}/{}/{} {}'.format(thread_id, atime, hex_id, nodeName)) multiThread(_updateRecord_httpGetWrapper, queue, maxWorkers=settings.MAX_CONNECTIONS) return True
def joinNetwork(msg): with Session() as s: queue = Queue() for node in Node.getInitNode(s).all(): queue.put((node.host,)) multiThread(_joinNetwork_findNodeWorker, queue, maxWorkers=settings.MAX_CONNECTIONS) with Session() as s: linkedNodeCount = Node.getLinkedNode(s).count() changeNodeCount = int(max(0, settings.MAX_NODES - linkedNodeCount) + settings.MAX_NODES/5) queue = Queue() for node in Node.getNotLinkedNode(s).limit(changeNodeCount).all(): queue.put((node.host,)) multiThread(_joinNetwork_joinWorker, queue, maxWorkers=settings.MAX_CONNECTIONS) with Session() as s: linkedNodes = Node.getLinkedNode(s).all() shuffle(linkedNodes) queue = Queue() for i in range(max(0, Node.getLinkedNode(s).count() - settings.MAX_NODES)): queue.put((linkedNodes[i].host,)) multiThread(_joinNetwork_doByeByeWorker, queue, maxWorkers=settings.MAX_CONNECTIONS)
def getThread(msg): log.isEnabledFor(logging.INFO) and log.info('getThread: {}'.format(msg.msg)) # ファイル名だけなら、全てのノードから順にスレッドを取得する if len(msg.msg.split()) == 1: fileName = msg.msg with Session() as s: for node in Node.getLinkedNode(s): MessageQueue.enqueue(s, msgtype='get_thread', msg=' '.join([ node.host, fileName, ])) s.commit() notify() return True with Session() as s: host, fileName = msg.msg.split() threadTitle = a2b_hex(fileName.split('_')[1]).decode('utf-8') thread = Thread.get(s, title=threadTitle).first() if thread is None: return lastTime = Record.getLastTime(s, thread.id) firstTime = Record.getFirstTime(s, thread.id) # 最新のレコードと、より古いレコードを取得する MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([ host, str(thread.id), str(lastTime)+'-', ])) if firstTime: MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([ host, str(thread.id), '-'+str(firstTime), ])) if firstTime and lastTime: # 未取得レコードが30%以上なら、その範囲をまとめて取得 # 30%未満なら、一つずつ取得 url = 'http://{}/head/{}/{}-{}'.format( host, fileName, str(firstTime), str(lastTime), ) records = [] notExistsRecordCount = 0 existsRecordCount = 0 _existsRecordCount = 0 rate = None try: for timestamp, recordId in str2recordInfo(httpGet(url)): timestamp = int(timestamp) if Record.get(s, thread.id, a2b_hex(recordId), timestamp).first(): if notExistsRecordCount: _existsRecordCount += 1 else: records.append((timestamp, recordId,)) notExistsRecordCount += 1 existsRecordCount += _existsRecordCount _existsRecordCount = 0 oldRate = rate rate = notExistsRecordCount / (notExistsRecordCount + existsRecordCount) if rate < 0.3 and oldRate >= 0.3: # records[0:-1]の範囲のレコードをまとめて取得 newRecords = records.pop() MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([ host, str(thread.id), str(records[0][0]) + '-' + str(records[-1][0]), ])) records = [newRecords] notExistsRecordCount = 1 existsRecordCount = 0 if rate is None: pass elif rate >= 0.3: # まとめて取得 MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([ host, str(thread.id), str(records[0][0]) + '-' + str(records[-1][0]), ])) else: # 一つずつ取得 for timestamp, recordId in records: MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([ host, str(thread.id), recordId, str(timestamp), ])) except URLError as e: log.isEnabledFor(logging.INFO) and log.info('getThread: Fail {} {} {}'.format(thread.id, url, str(e))) s.commit() notify()