Esempio n. 1
0
	def post(self, request, *args, **kwargs):
		try:
			thread_id = int(kwargs['thread_id'])
			name = request.POST['name']
			mail = request.POST['mail']
			body = request.POST['body']
			try:
				attach = request.POST['attach']
				attach_sfx = request.POST['attach_sfx']
			except KeyError:
				attach = None
				attach_sfx = None
		except KeyError:
			return HttpResponseBadRequest()

		timestamp = time.time()
		try:
			recStr = makeRecordStr(timestamp, name, mail, body, attach, attach_sfx)
		except BadRecordException:
			return HttpResponseBadRequest()
		_, bin_id_hex, body = tuple(str2recordInfo(recStr))[0]
		bin_id = a2b_hex(bin_id_hex)
		with Session() as s:
			try:
				Record.add(s, thread_id, timestamp, bin_id, body)
				s.commit()
				threadTitle = Thread.get(s, id=thread_id).value(Thread.title)
				Recent.add(s, timestamp, bin_id, Thread.getFileName(threadTitle))
				s.commit()
				msgqueue.updateRecord(thread_id, bin_id_hex, timestamp)
			except IntegrityError:
				s.rollback()
		return HttpResponse()
Esempio n. 2
0
def updateRecord(msg):
	with Session() as s:
		thread_id, hex_id, atime, nodeName = msg.msg.split()
		thread_id = int(thread_id)
		atime = int(atime)
		filename = Thread.getFileName(Thread.get(s, id=thread_id).value(Thread.title))

		queue = Queue()
		for host in Node.getLinkedNode(s).values(Node.host):
			queue.put((
				host.host, filename, atime, hex_id, thread_id, nodeName
				))
	log.isEnabledFor(logging.INFO) and log.info('updateRecord: Run {}/{}/{} {}'.format(thread_id, atime, hex_id, nodeName))
	multiThread(_updateRecord_httpGetWrapper, queue, maxWorkers=settings.MAX_CONNECTIONS)
	return True
Esempio n. 3
0
	def head(self, request, *args, **kwargs):
		with Session() as s:
			for t in Thread.gets(s, title=request.GET.get('title')):
				if t.records >=1:
					return HttpResponse()
				break
		return HttpResponseNotFound()
Esempio n. 4
0
	def dispatch(self, request, *args, **kwargs):
		with Session() as s:
			try:
				fileName = kwargs['file']
				prefix, basename = splitFileName(kwargs['file'])
				title = a2b_hex(basename)
				atime = int(kwargs['time'])
				id_hex = kwargs['id']
				id_bin = a2b_hex(id_hex)
				addr = kwargs['node'].replace('+', '/')
			except (BadFileNameException, binascii.Error, ValueError):
				return HttpResponseBadRequest()

			if addr.startswith(':') or addr.startswith('/'):
				addr = request.META['REMOTE_ADDR'] + addr
			response = HttpResponse()
			if prefix=='thread':
				if Recent.get(s, atime, id_bin, fileName).first():
					return response
				thread_id = Thread.get(s, title=title).value(Thread.id)
				if thread_id:
					msgqueue.getAndUpdateRecord(addr, thread_id, id_hex, atime)
				try:
					Recent.add(s, timestamp=atime, binId=id_bin, fileName=fileName)
					s.commit()
				except IntegrityError:
					s.rollback()
			return response
Esempio n. 5
0
def _getRecent_worker(host):
	urlSuffix = '/recent/0-'
	try:
		recent = httpGet('http://' + host + urlSuffix)
	except URLError:
		return
	with Session() as s:
		fileNames = set()
		for line in str2recordInfo(recent):
			timestamp, recordId, fileName = line[0:3]
			if fileName.split('_')[0] not in ('thread'):
				continue
			fileNames.add(fileName)

		for fileName in sorted(fileNames):
			try:
				threadTitle = a2b_hex(fileName.split('_')[1])
			except (ValueError, binascii.Error):
				continue
			thread = Thread.get(s, title=threadTitle).first()
			if thread is None:
				continue
			log.isEnabledFor(logging.INFO) and log.info('getRecent: found {} {}'.format(thread.id, host))
			MessageQueue.enqueue(s, msgtype='get_thread', msg=' '.join((host, fileName)))
		s.commit()
	notify()
Esempio n. 6
0
def getRecord(msg):
	with Session() as s:
		if len(msg.msg.split()) == 4:
			mode = 'single'
			addr, thread_id, hex_id, atime = msg.msg.split()
			thread_id = int(thread_id)
			bin_id = a2b_hex(hex_id)
			atime = int(atime)
		else:
			mode = 'multi'
			addr, thread_id, timeRange = msg.msg.split()

		if mode == 'single' and Record.get(s, thread_id, bin_id, atime).value(Record.record_id):
			log.isEnabledFor(logging.INFO) and log.info('getRecord: NOP {}/{}/{} {}'.format(thread_id, atime, hex_id, addr))
			return True

		title = Thread.get(s, id=thread_id).value(Thread.title)
		filename = Thread.getFileName(title)
		if mode == 'single':
			http_addr = 'http://{}/get/{}/{}'.format(addr, filename, atime)
		else:
			http_addr = 'http://{}/get/{}/{}'.format(addr, filename, timeRange)
		try:
			for record in str2recordInfo(httpGet(http_addr)):
				try:
					timestamp, hex_id, body = record
					bin_id = a2b_hex(hex_id)
				except binascii.Error as e:
					log.isEnabledFor(logging.INFO) and log.info('getRecord: Fail {} {} {}'.format(thread_id, http_addr, str(e)))
					continue
				try:
					timestamp = int(timestamp)
					if Record.get(s, thread_id, bin_id, timestamp).first():
						continue
					Record.add(s, thread_id, timestamp, bin_id, body)
					log.isEnabledFor(logging.INFO) and log.info('getRecord: Add {}/{}/{} {}'.format(thread_id, timestamp, b2a_hex(bin_id), addr))
					s.commit()
				except (binascii.Error, sqlalchemy.exc.StatementError) as e:
					log.isEnabledFor(logging.INFO) and log.info('getRecord: Fail {} {} {}'.format(thread_id, http_addr, str(e)))
					s.rollback()
					Record.delete(s, thread_id, timestamp, bin_id)
					s.commit()
		except URLError as e:
			log.isEnabledFor(logging.INFO) and log.info('getRecord: Fail {} {} {}'.format(thread_id, http_addr, str(e)))
			return False
		return True
Esempio n. 7
0
	def post(self, request, *args, **kwargs):
		title = request.POST['title']
		with Session() as s:
			query = Thread.get(s, title=title).all()
			if len(query) == 0:
				thread = Thread.add(s, title)
				MessageQueue.enqueue(s, msgtype='get_thread', msg=Thread.getFileName(title))
				s.commit()
				notify()
			else:
				thread = query[0]
			return JsonResponse({
				'thread': {
					"id": thread.id,
					"title": thread.title,
					"timestamp": thread.timestamp,
					"records": thread.records,
					}
				})
Esempio n. 8
0
	def dispatch(self, request, *args, **kwargs):
		with Session() as s:
			try:
				prefix, basename = splitFileName(kwargs['file'])
			except BadFileNameException:
				return HttpResponseBadRequest()

			response = HttpResponse()
			if prefix=='thread':
				try:
					title = a2b_hex(basename)
				except (TypeError, binascii.Error):
					return HttpResponseBadRequest()
				if Thread.get(s, title=title).value(Thread.id):
					response.write('YES')
					return response
			response.write('NO')
			return response
Esempio n. 9
0
	def get(self, request, *args, **kwargs):
		threads = []
		with Session() as s:
			result = Thread.gets(s,
					limit=intOrNone(request.GET.get('limit')),
					stime=intOrNone(request.GET.get('start_time')),
					etime=intOrNone(request.GET.get('end_time')),
					title=request.GET.get('title'),
					)
			for t in result:
				threads.append({
					'id': int(t.id),
					'title': t.title,
					'timestamp': int(datetime2timestamp(t.timestamp)),
					'records': int(t.records),
					})
		obj = {
				'threads': threads
				}
		return JsonResponse(obj)
Esempio n. 10
0
	def dispatch(self, request, *args, **kwargs):
		with Session() as s:
			try:
				prefix, basename = splitFileName(kwargs['file'])
				stime, etime = getTimeRange(
						kwargs.get('time'),
						kwargs.get('stime'),
						kwargs.get('etime'),
						)
			except (BadFileNameException, BadTimeRange):
				return HttpResponseBadRequest()

			response = HttpResponse()
			if prefix=='thread':
				try:
					title = a2b_hex(basename)
				except binascii.Error:
					return HttpResponseBadRequest()
				thread_id = Thread.get(s, title=title).value(Thread.id)
				allRecords = Record.gets(s, thread_id, stime, etime)
				for line in record2str(allRecords, 1):
					response.write(line)
			return response
Esempio n. 11
0
def getThread(msg):
	log.isEnabledFor(logging.INFO) and log.info('getThread: {}'.format(msg.msg))

	# ファイル名だけなら、全てのノードから順にスレッドを取得する
	if len(msg.msg.split()) == 1:
		fileName = msg.msg
		with Session() as s:
			for node in Node.getLinkedNode(s):
				MessageQueue.enqueue(s, msgtype='get_thread', msg=' '.join([
					node.host,
					fileName,
					]))
			s.commit()
		notify()
		return True

	with Session() as s:
		host, fileName = msg.msg.split()

		threadTitle = a2b_hex(fileName.split('_')[1]).decode('utf-8')
		thread = Thread.get(s, title=threadTitle).first()
		if thread is None:
			return

		lastTime = Record.getLastTime(s, thread.id)
		firstTime = Record.getFirstTime(s, thread.id)

		# 最新のレコードと、より古いレコードを取得する
		MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([
			host, str(thread.id),
			str(lastTime)+'-',
			]))
		if firstTime:
			MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([
				host, str(thread.id),
				'-'+str(firstTime),
				]))

		if firstTime and lastTime:
			# 未取得レコードが30%以上なら、その範囲をまとめて取得
			# 30%未満なら、一つずつ取得
			url = 'http://{}/head/{}/{}-{}'.format(
					host, fileName,
					str(firstTime),
					str(lastTime),
					)
			records = []
			notExistsRecordCount = 0
			existsRecordCount = 0
			_existsRecordCount = 0
			rate = None
			try:
				for timestamp, recordId in str2recordInfo(httpGet(url)):
					timestamp = int(timestamp)
					if Record.get(s, thread.id, a2b_hex(recordId), timestamp).first():
						if notExistsRecordCount:
							_existsRecordCount += 1
					else:
						records.append((timestamp, recordId,))
						notExistsRecordCount += 1
						existsRecordCount += _existsRecordCount
						_existsRecordCount = 0
						oldRate = rate
						rate = notExistsRecordCount / (notExistsRecordCount + existsRecordCount)

						if rate < 0.3 and oldRate >= 0.3:
							# records[0:-1]の範囲のレコードをまとめて取得
							newRecords = records.pop()
							MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([
								host, str(thread.id),
								str(records[0][0]) + '-' + str(records[-1][0]),
								]))
							records = [newRecords]
							notExistsRecordCount = 1
							existsRecordCount = 0

				if rate is None:
					pass
				elif rate >= 0.3:
					# まとめて取得
					MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([
						host, str(thread.id),
						str(records[0][0]) + '-' + str(records[-1][0]),
						]))
				else:
					# 一つずつ取得
					for timestamp, recordId in records:
						MessageQueue.enqueue(s, msgtype='get_record', msg=' '.join([
							host, str(thread.id), recordId, str(timestamp),
							]))
			except URLError as e:
					log.isEnabledFor(logging.INFO) and log.info('getThread: Fail {} {} {}'.format(thread.id, url, str(e)))

		s.commit()
		notify()