def post(self, request, *args, **kwargs): try: thread_id = int(kwargs['thread_id']) name = request.POST['name'] mail = request.POST['mail'] body = request.POST['body'] try: attach = request.POST['attach'] attach_sfx = request.POST['attach_sfx'] except KeyError: attach = None attach_sfx = None except KeyError: return HttpResponseBadRequest() timestamp = time.time() try: recStr = makeRecordStr(timestamp, name, mail, body, attach, attach_sfx) except BadRecordException: return HttpResponseBadRequest() _, bin_id_hex, body = tuple(str2recordInfo(recStr))[0] bin_id = a2b_hex(bin_id_hex) with Session() as s: try: Record.add(s, thread_id, timestamp, bin_id, body) s.commit() threadTitle = Thread.get(s, id=thread_id).value(Thread.title) Recent.add(s, timestamp, bin_id, Thread.getFileName(threadTitle)) s.commit() msgqueue.updateRecord(thread_id, bin_id_hex, timestamp) except IntegrityError: s.rollback() return HttpResponse()
def head(self, request, *args, **kwargs): try: thread_id = int(kwargs['thread_id']) bin_id = a2b_hex(kwargs['record_id']) timestamp = int(kwargs['timestamp']) except KeyError: return HttpResponseBadRequest() with Session() as s: if Record.get(s, thread_id, bin_id, timestamp).with_entities(Record.bin_id).first(): return HttpResponse() return HttpResponseNotFound()
def getRecord(msg):
with Session() as s:
if len(msg.msg.split()) == 4:
mode = 'single'
addr, thread_id, hex_id, atime = msg.msg.split()
thread_id = int(thread_id)
bin_id = a2b_hex(hex_id)
atime = int(atime)
else:
mode = 'multi'
addr, thread_id, timeRange = msg.msg.split()
if mode == 'single' and Record.get(s, thread_id, bin_id, atime).value(Record.record_id):
log.isEnabledFor(logging.INFO) and log.info('getRecord: NOP {}/{}/{} {}'.format(thread_id, atime, hex_id, addr))
return True
title = Thread.get(s, id=thread_id).value(Thread.title)
filename = Thread.getFileName(title)
if mode == 'single':
http_addr = 'http://{}/get/{}/{}'.format(addr, filename, atime)
else:
http_addr = 'http://{}/get/{}/{}'.format(addr, filename, timeRange)
try:
for record in str2recordInfo(httpGet(http_addr)):
try:
timestamp, hex_id, body = record
bin_id = a2b_hex(hex_id)
except binascii.Error as e:
log.isEnabledFor(logging.INFO) and log.info('getRecord: Fail {} {} {}'.format(thread_id, http_addr, str(e)))
continue
try:
timestamp = int(timestamp)
if Record.get(s, thread_id, bin_id, timestamp).first():
continue
Record.add(s, thread_id, timestamp, bin_id, body)
log.isEnabledFor(logging.INFO) and log.info('getRecord: Add {}/{}/{} {}'.format(thread_id, timestamp, b2a_hex(bin_id), addr))
s.commit()
except (binascii.Error, sqlalchemy.exc.StatementError) as e:
log.isEnabledFor(logging.INFO) and log.info('getRecord: Fail {} {} {}'.format(thread_id, http_addr, str(e)))
s.rollback()
Record.delete(s, thread_id, timestamp, bin_id)
s.commit()
except URLError as e:
log.isEnabledFor(logging.INFO) and log.info('getRecord: Fail {} {} {}'.format(thread_id, http_addr, str(e)))
return False
return True
def dispatch(self, request, *args, **kwargs):
with Session() as s:
try:
prefix, basename = splitFileName(kwargs['file'])
stime, etime = getTimeRange(
kwargs.get('time'),
kwargs.get('stime'),
kwargs.get('etime'),
)
except (BadFileNameException, BadTimeRange):
return HttpResponseBadRequest()
response = HttpResponse()
if prefix=='thread':
try:
title = a2b_hex(basename)
except binascii.Error:
return HttpResponseBadRequest()
thread_id = Thread.get(s, title=title).value(Thread.id)
allRecords = Record.gets(s, thread_id, stime, etime)
for line in record2str(allRecords, 1):
response.write(line)
return response
def get(self, request, *args, **kwargs):
records = []
thread_id = int(kwargs['thread_id'])
with Session() as s:
if 'timestamp' in kwargs:
""" 単一のレコードを返す方のAPI """
timestamp = int(kwargs['timestamp'])
bin_id = a2b_hex(kwargs['record_id'])
r = Record.get(s, thread_id, bin_id, timestamp).with_entities(
Record.bin_id,
Record.timestamp,
Record.name,
Record.mail,
Record.body,
sql_func.length(Record.attach).label('attach_len'),
Record.suffix).first()
if r:
records.append({
'id': b2a_hex(r.bin_id).decode('ascii'),
'timestamp': int(datetime2timestamp(r.timestamp)),
'name': r.name,
'mail': r.mail,
'body': r.body,
'attach': bool(r.attach_len),
'suffix': r.suffix,
})
else:
""" 複数のレコードを返す方のAPI """
bin_id = request.GET.get('record_id')
if bin_id:
bin_id = a2b_hex(bin_id)
matchRecords = Record.gets(s,
thread_id=thread_id,
stime=intOrNone(request.GET.get('start_time')),
etime=intOrNone(request.GET.get('end_time')),
bin_id=bin_id,
limit=intOrNone(request.GET.get('limit')),
).with_entities(
Record.bin_id,
Record.timestamp,
Record.name,
Record.mail,
Record.body,
sql_func.length(Record.attach).label('attach_len'),
Record.suffix)
for r in matchRecords:
records.append({
'id': b2a_hex(r.bin_id).decode('ascii'),
'timestamp': int(datetime2timestamp(r.timestamp)),
'name': r.name,
'mail': r.mail,
'body': r.body,
'attach': bool(r.attach_len),
'suffix': r.suffix,
})
obj = {
'records': records,
}
return JsonResponse(obj)
def getThread(msg):
log.isEnabledFor(logging.INFO) and log.info('getThread: {}'.format(msg.msg))
# ファイル名だけなら、全てのノードから順にスレッドを取得する
if len(msg.msg.split()) == 1:
fileName = msg.msg
with Session() as s:
for node in Node.getLinkedNode(s):
MessageQueue.enqueue(s, msgtype='get_thread', msg=' '.join([
node.host,
fileName,
]))
s.commit()
notify()
return True
with Session() as s:
host, fileName = msg.msg.split()
threadTitle = a2b_hex(fileName.split('_')[1]).decode('utf-8')
thread = Thread.get(s, title=threadTitle).first()
if thread is None:
return
lastTime = Record.getLastTime(s, thread.id)
firstTime = Record.getFirstTime(s, thread.id)
# 最新のレコードと、より古いレコードを取得する
MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([
host, str(thread.id),
str(lastTime)+'-',
]))
if firstTime:
MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([
host, str(thread.id),
'-'+str(firstTime),
]))
if firstTime and lastTime:
# 未取得レコードが30%以上なら、その範囲をまとめて取得
# 30%未満なら、一つずつ取得
url = 'http://{}/head/{}/{}-{}'.format(
host, fileName,
str(firstTime),
str(lastTime),
)
records = []
notExistsRecordCount = 0
existsRecordCount = 0
_existsRecordCount = 0
rate = None
try:
for timestamp, recordId in str2recordInfo(httpGet(url)):
timestamp = int(timestamp)
if Record.get(s, thread.id, a2b_hex(recordId), timestamp).first():
if notExistsRecordCount:
_existsRecordCount += 1
else:
records.append((timestamp, recordId,))
notExistsRecordCount += 1
existsRecordCount += _existsRecordCount
_existsRecordCount = 0
oldRate = rate
rate = notExistsRecordCount / (notExistsRecordCount + existsRecordCount)
if rate < 0.3 and oldRate >= 0.3:
# records[0:-1]の範囲のレコードをまとめて取得
newRecords = records.pop()
MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([
host, str(thread.id),
str(records[0][0]) + '-' + str(records[-1][0]),
]))
records = [newRecords]
notExistsRecordCount = 1
existsRecordCount = 0
if rate is None:
pass
elif rate >= 0.3:
# まとめて取得
MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([
host, str(thread.id),
str(records[0][0]) + '-' + str(records[-1][0]),
]))
else:
# 一つずつ取得
for timestamp, recordId in records:
MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([
host, str(thread.id), recordId, str(timestamp),
]))
except URLError as e:
log.isEnabledFor(logging.INFO) and log.info('getThread: Fail {} {} {}'.format(thread.id, url, str(e)))
s.commit()
notify()