def _getRecent_worker(host):
urlSuffix = '/recent/0-'
try:
recent = httpGet('http://' + host + urlSuffix)
except URLError:
return
with Session() as s:
fileNames = set()
for line in str2recordInfo(recent):
timestamp, recordId, fileName = line[0:3]
if fileName.split('_')[0] not in ('thread'):
continue
fileNames.add(fileName)
for fileName in sorted(fileNames):
try:
threadTitle = a2b_hex(fileName.split('_')[1])
except (ValueError, binascii.Error):
continue
thread = Thread.get(s, title=threadTitle).first()
if thread is None:
continue
log.isEnabledFor(logging.INFO) and log.info('getRecent: found {} {}'.format(thread.id, host))
MessageQueue.enqueue(s, msgtype='get_thread', msg=' '.join((host, fileName)))
s.commit()
notify()
def doPing(msg): with Session() as s: queue = Queue() for node in Node.getLinkedNode(s).all(): queue.put((node.host,)) multiThread(_doPing_worker, queue, maxWorkers=settings.MAX_CONNECTIONS) MessageQueue.enqueue(s, msgtype='join', msg='init')
def timer():
if wait:
time.sleep(wait)
while True:
with Session() as s:
log.isEnabledFor(logging.DEBUG) and log.debug(
'timer[{}] {}'.format(msgtype, msg))
MessageQueue.enqueue(s, msgtype=msgtype, msg=msg)
s.commit()
notify()
if not interval:
break
time.sleep(interval)
def updateRecord(thread_id, hex_id, timestamp, nodeName=settings.NODE_NAME):
log.isEnabledFor(logging.INFO) and log.info(
'updateRecord: threadId:{} hexId:{} timestamp:{}'.format(thread_id, hex_id, timestamp))
with Session() as s:
timestamp = str(int(float(timestamp)))
hex_id = str(hex_id)
thread_id = str(thread_id)
msg = ' '.join((thread_id, hex_id, timestamp, nodeName))
MessageQueue.enqueue(s, msgtype='update_record', msg=msg)
s.commit()
notify()
def getAndUpdateRecord(addr, thread_id, hex_id, timestamp):
log.isEnabledFor(logging.INFO) and log.info(
'getAndUpdateRecord: threadId:{} hexId:{} timestamp:{}'.format(thread_id, hex_id, timestamp))
with Session() as s:
timestamp = int(float(timestamp))
msg = ' '.join((addr, str(thread_id), str(hex_id), str(timestamp)))
if addr is not None:
MessageQueue.enqueue(s, msgtype='get_record', msg=msg)
msg = ' '.join((str(thread_id), str(hex_id), str(timestamp), settings.NODE_NAME))
MessageQueue.enqueue(s, msgtype='update_record', msg=msg)
s.commit()
notify()
def post(self, request, *args, **kwargs):
title = request.POST['title']
with Session() as s:
query = Thread.get(s, title=title).all()
if len(query) == 0:
thread = Thread.add(s, title)
MessageQueue.enqueue(s, msgtype='get_thread', msg=Thread.getFileName(title))
s.commit()
notify()
else:
thread = query[0]
return JsonResponse({
'thread': {
"id": thread.id,
"title": thread.title,
"timestamp": thread.timestamp,
"records": thread.records,
}
})
def dispatcher(workerFunc):
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
try:
os.remove(settings.MESSAGE_QUEUE_SOCK_FILE)
except FileNotFoundError:
pass
sock.bind(settings.MESSAGE_QUEUE_SOCK_FILE)
sock.settimeout(10)
sock.listen(1)
while True:
try:
with Session() as s:
msg = MessageQueue.dequeue(s)
s.commit()
if msg is None:
s.close()
try:
conn, addr = sock.accept()
conn.close()
except KeyboardInterrupt:
return
except socket.timeout:
pass
continue
log.isEnabledFor(logging.DEBUG) and log.debug(' '.join([msg.getTypeName(s), msg.msg]))
worker = workerFunc.get(msg.getTypeName(s))
try:
worker(msg)
except Exception:
log.error('\n'.join([
'Worker died.',
traceback.format_exc(),
]))
except Exception:
log.critical('\n'.join([
'Error occurs in the message loop...',
traceback.format_exc(),
]))
time.sleep(3)
def getThread(msg):
log.isEnabledFor(logging.INFO) and log.info('getThread: {}'.format(msg.msg))
# ファイル名だけなら、全てのノードから順にスレッドを取得する
if len(msg.msg.split()) == 1:
fileName = msg.msg
with Session() as s:
for node in Node.getLinkedNode(s):
MessageQueue.enqueue(s, msgtype='get_thread', msg=' '.join([
node.host,
fileName,
]))
s.commit()
notify()
return True
with Session() as s:
host, fileName = msg.msg.split()
threadTitle = a2b_hex(fileName.split('_')[1]).decode('utf-8')
thread = Thread.get(s, title=threadTitle).first()
if thread is None:
return
lastTime = Record.getLastTime(s, thread.id)
firstTime = Record.getFirstTime(s, thread.id)
# 最新のレコードと、より古いレコードを取得する
MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([
host, str(thread.id),
str(lastTime)+'-',
]))
if firstTime:
MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([
host, str(thread.id),
'-'+str(firstTime),
]))
if firstTime and lastTime:
# 未取得レコードが30%以上なら、その範囲をまとめて取得
# 30%未満なら、一つずつ取得
url = 'http://{}/head/{}/{}-{}'.format(
host, fileName,
str(firstTime),
str(lastTime),
)
records = []
notExistsRecordCount = 0
existsRecordCount = 0
_existsRecordCount = 0
rate = None
try:
for timestamp, recordId in str2recordInfo(httpGet(url)):
timestamp = int(timestamp)
if Record.get(s, thread.id, a2b_hex(recordId), timestamp).first():
if notExistsRecordCount:
_existsRecordCount += 1
else:
records.append((timestamp, recordId,))
notExistsRecordCount += 1
existsRecordCount += _existsRecordCount
_existsRecordCount = 0
oldRate = rate
rate = notExistsRecordCount / (notExistsRecordCount + existsRecordCount)
if rate < 0.3 and oldRate >= 0.3:
# records[0:-1]の範囲のレコードをまとめて取得
newRecords = records.pop()
MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([
host, str(thread.id),
str(records[0][0]) + '-' + str(records[-1][0]),
]))
records = [newRecords]
notExistsRecordCount = 1
existsRecordCount = 0
if rate is None:
pass
elif rate >= 0.3:
# まとめて取得
MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([
host, str(thread.id),
str(records[0][0]) + '-' + str(records[-1][0]),
]))
else:
# 一つずつ取得
for timestamp, recordId in records:
MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([
host, str(thread.id), recordId, str(timestamp),
]))
except URLError as e:
log.isEnabledFor(logging.INFO) and log.info('getThread: Fail {} {} {}'.format(thread.id, url, str(e)))
s.commit()
notify()
