def fetch(cmd, **kwargs):
g = Groonga()
ret = g.call(cmd, **kwargs)
print ret.status
print ret.body
print "*" * 40
return ret.body
def fetch(cmd, **kwargs):
g = Groonga()
ret = g.call(cmd, **kwargs)
print(ret.status)
print(ret.body)
print("*" * 40)
return ret.body
def fetch(cmd, **kwargs):
g = Groonga()
ret = g.call(cmd, **kwargs)
print ret.status
print ret.body
print "*" * 40
return ret.body
def test_convert_gqtp_result(self):
grn = Groonga()
s = grn._clock_gettime()
e = grn._clock_gettime()
senddata = get_send_data_for_gqtp("status")
rawdata = senddata[:GQTP_HEADER_SIZE] + b'{"test": 0}'
d = convert_gqtp_result_data(s, e, 0, rawdata)
d = json.loads(d)
self.assertEqual(d[0][0], 0)
def test_convert_gqtp_result_fail(self):
grn = Groonga()
s = grn._clock_gettime()
e = grn._clock_gettime()
senddata = get_send_data_for_gqtp("status")
rawdata = senddata[:GQTP_HEADER_SIZE] + b'{"test": 0}'
d = convert_gqtp_result_data(s, e, GRN_STATUS_UNSUPPORTED_COMMAND_VERSION, rawdata)
d = json.loads(d)
self.assertEqual(d[0][0], -71)
def test_convert_gqtp_result_fail(self):
grn = Groonga()
s = grn._clock_gettime()
e = grn._clock_gettime()
senddata = get_send_data_for_gqtp("status")
rawdata = senddata[:GQTP_HEADER_SIZE] + b"{\"test\": 0}"
d = convert_gqtp_result_data(s, e, GRN_STATUS_UNSUPPORTED_COMMAND_VERSION, rawdata)
d = json.loads(d)
self.assertEqual(d[0][0], -71)
def test_convert_gqtp_result(self):
grn = Groonga()
s = grn._clock_gettime()
e = grn._clock_gettime()
senddata = get_send_data_for_gqtp("status")
rawdata = senddata[:GQTP_HEADER_SIZE] + b"{\"test\": 0}"
d = convert_gqtp_result_data(s, e, 0, rawdata)
d = json.loads(d)
self.assertEqual(d[0][0], 0)
def _load(path):
grn = Groonga()
filename = os.path.basename(path)
ext = os.path.splitext(filename)[1]
content = open(path).read()
data = """[{"_key":"%s","name":"%s","content":"%s","type":"%s"}]""" % (
path, filename, re.escape(content), ext)
data = [{"_key": path, "name": filename, "content": content, "ext": ext}]
data = json.dumps(data)
ret = grn.call("load", table="Files", values=data)
print ret.body
class PoyongaHTTPTestCase(unittest.TestCase):
def setUp(self):
self.g = Groonga()
@patch("poyonga.client.urlopen")
def test_json_result_with_http(self, mock_urlopen):
m = Mock()
m.read.side_effect = ["[[0, 1337566253.89858, 0.000354], {}]"]
mock_urlopen.return_value = m
ret = self.g.call("status")
self.assertEqual(type(ret), GroongaResult)
self.assertEqual(ret.status, 0)
class PoyongaHTTPTestCase(unittest.TestCase):
def setUp(self):
self.g = Groonga()
@patch('poyonga.client.urlopen')
def test_json_result_with_http(self, mock_urlopen):
m = Mock()
m.read.side_effect = ['[[0, 1337566253.89858, 0.000354], {}]']
mock_urlopen.return_value = m
ret = self.g.call('status')
self.assertEqual(type(ret), GroongaResult)
self.assertEqual(ret.status, 0)
class PoyongaGQTPTestCase(unittest.TestCase):
def setUp(self):
self.g = Groonga(protocol='gqtp')
@patch('poyonga.client.socket.socket')
def test_json_result_with_gqtp(self, mock_socket):
m = Mock()
if sys.version_info[0] == 3:
_data = b"{}"
else:
_data = "{}"
_proto, _qtype, _keylen, _level, _flags, _status, _size, _opaque, _cas = \
0xc7, 0x02, 0, 0, 0, 0, 2, 0, 0
packdata = struct.pack("!BBHBBHIIQ2s", _proto, _qtype, _keylen, _level,
_flags, _status, _size, _opaque, _cas, _data)
m.recv.return_value = packdata
mock_socket.return_value = m
ret = self.g.call('status')
self.assertEqual(type(ret), GroongaResult)
self.assertEqual(ret.status, 0)
class PoyongaGQTPTestCase(unittest.TestCase):
def setUp(self):
self.g = Groonga(protocol='gqtp')
@patch('poyonga.client.socket.socket')
def test_json_result_with_gqtp(self, mock_socket):
m = Mock()
if sys.version_info[0] == 3:
_data = b"{}"
else:
_data = "{}"
_proto, _qtype, _keylen, _level, _flags, _status, _size, _opaque, _cas = \
0xc7, 0x02, 0, 0, 0, 0, 2, 0, 0
packdata = struct.pack("!BBHBBHIIQ2s",
_proto, _qtype, _keylen, _level, _flags, _status, _size, _opaque, _cas, _data)
m.recv.return_value = packdata
mock_socket.return_value = m
ret = self.g.call('status')
self.assertEqual(type(ret), GroongaResult)
self.assertEqual(ret.status, 0)
def setUp(self):
self.g = Groonga(protocol="gqtp")
from poyonga import Groonga
g = Groonga(protocol="gqtp", port=10043)
cmds = [
# table_remove
("table_remove", {
"name": "Writers",
"dependent": "yes"
}),
("table_remove", {
"name": "Months",
"dependent": "yes"
}),
("table_remove", {
"name": "Types",
"dependent": "yes"
}),
("table_remove", {
"name": "Backgrounds",
"dependent": "yes"
}),
("table_remove", {
"name": "Expressions",
"dependent": "yes"
}),
("table_remove", {
"name": "Scenario",
"dependent": "yes"
}),
("table_remove", {
def setUp(self):
self.g = Groonga()
import sys
from poyonga import Groonga
query = "content:@%s" % sys.argv[1]
grn = Groonga()
ret = grn.call("select", table="Files", limit=1000,
sortby="-_score", output_columns="_score,_key", query=query)
for item in ret.items:
print "[score:%3d]%s" % (item['_score'], item['_key'])
print "hit:", ret.hit_num
def setUp(self):
self.g = Groonga(protocol='gqtp')
from poyonga import Groonga
grn = Groonga()
ret = grn.call("table_create", name="Files", flags="TABLE_HASH_KEY",
key_type="ShortText")
print ret.status
ret = grn.call("table_create", name="Bigram", key_type="ShortText",
flags="TABLE_PAT_KEY|KEY_NORMALIZE",
default_tokenizer="TokenBigramSplitSymbolAlphaDigit")
print ret.status
ret = grn.call("column_create", table="Files", name="ext",
flags="COLUMN_SCALAR", type="ShortText")
print ret.status
ret = grn.call("column_create", table="Files", name="name",
flags="COLUMN_SCALAR", type="ShortText")
print ret.status
ret = grn.call("column_create", table="Files", name="content",
flags="COLUMN_SCALAR", type="ShortText")
print ret.status
ret = grn.call("column_create", table="Bigram", name="content_index",
flags="COLUMN_INDEX|WITH_POSITION", type="Files",
source="content")
print ret.status
def kmsksearch():
'''
検索ページ
'''
# 初期化
PER_PAGE = 10
DRILLDROWN_LIMIT = 999999
request_dict = {}
groonga_query_dict = {}
query_para_dict = {}
filter_dict = {}
view_filter_list = []
# フィルタ追加
app.jinja_env.filters['delparameter'] = delparameter
app.jinja_env.filters['addparameter'] = addparameter
# リクエストパラメーター入れ替え
request_dict['page'] = request.args.get(get_page_parameter(), 1, type=int)
request_dict['word_no_sanitize'] = request.args.get('word', '', type=str)
request_dict['word'] = escape(request.args.get('word', '', type=str))
request_dict['writer'] = escape(request.args.get('writer', '', type=str))
request_dict['type'] = escape(request.args.get('type', '', type=str))
request_dict['month'] = escape(request.args.get('month', '', type=str))
request_dict['speaker'] = escape(request.args.get('speaker', '', type=str))
request_dict['background'] = escape(
request.args.get('background', '', type=str))
request_dict['expression'] = escape(
request.args.get('expression', '', type=str))
request_dict['cancel'] = escape(request.args.get('cancel', '', type=str))
# 検索ワード存在チェックと禁止文字チェック
if not request_dict['word']:
return render_template('_index.html', message="検索キーワードを指定してください。")
query_para_dict['word'] = request_dict['word']
# グルンガのメインクエリ作成
groonga_query_dict["table"] = "Scenario"
groonga_query_dict[
"output_columns"] = "id.capter,id.title,id.subtitle,id.episode,speaker,highlight_html(text),taps,background.name,id.type.name,expression.name,id.month.name"
groonga_query_dict["sort_keys"] = "id.capter,id.order,taps"
groonga_query_dict["command_version"] = 2
groonga_query_dict["limit"] = PER_PAGE
groonga_query_dict["drilldowns[expression].keys"] = "expression.name"
groonga_query_dict["drilldowns[expression].sort_keys"] = "_key"
groonga_query_dict["drilldowns[expression].limit"] = DRILLDROWN_LIMIT
groonga_query_dict["drilldowns[speaker].keys"] = "speaker"
groonga_query_dict["drilldowns[speaker].sort_keys"] = "_key"
groonga_query_dict["drilldowns[speaker].limit"] = DRILLDROWN_LIMIT
groonga_query_dict[
"drilldowns[writer].keys"] = "id.writer._key,id.writer.name"
groonga_query_dict[
"drilldowns[writer].output_columns"] = "_value.id.writer._key,_value.id.writer.name,_nsubrecs"
groonga_query_dict[
"drilldowns[writer].sort_keys"] = "_value.id.writer._key"
groonga_query_dict["drilldowns[writer].limit"] = DRILLDROWN_LIMIT
groonga_query_dict[
"drilldowns[month].keys"] = "id.month._key,id.month.name"
groonga_query_dict[
"drilldowns[month].output_columns"] = "_value.id.month._key,_value.id.month.name,_nsubrecs"
groonga_query_dict["drilldowns[month].sort_keys"] = "_value.id.month._key"
groonga_query_dict["drilldowns[month].limit"] = DRILLDROWN_LIMIT
groonga_query_dict["drilldowns[type].keys"] = "id.type._key,id.type.name"
groonga_query_dict[
"drilldowns[type].output_columns"] = "_value.id.type._key,_value.id.type.name,_nsubrecs"
groonga_query_dict["drilldowns[type].sort_keys"] = "_value.id.type._key"
groonga_query_dict["drilldowns[type].limit"] = DRILLDROWN_LIMIT
# groonga_query_dict["drilldowns[background].keys"] = "background._key,background.name"
# groonga_query_dict["drilldowns[background].output_columns"] = "_value.background._key,_value.background.name,_nsubrecs"
# groonga_query_dict["drilldowns[background].sort_keys"] = "_value.background._key"
# groonga_query_dict["drilldowns[background].limit"] = DRILLDROWN_LIMIT
# ANDクエリと禁止文字チェック
querystring = ''
forbidden_char_list = ['、', '。', '「', '」', '（', '）', '(', ')']
for i, v in enumerate(request_dict['word_no_sanitize'].strip().split()):
# ハピエレと日日日先生に怒られるので、一文字で以下の文字は検索させない
# 全角：、。「」（）
# 半角：()
if len(v) == 1:
if v in forbidden_char_list:
return render_template('_index.html',
message="検索できない文字が含まれています。")
if i > 0:
querystring += ' + '
querystring += 'text:@"{}"'.format(v)
groonga_query_dict["query"] = querystring
# グルンガのフィルタクエリ前処理(全解除の場合処理はしない)
filter_pattern_dict = {
'writer': 'id.writer.name == "{}"',
'type': 'id.type.name == "{}"',
'month': 'id.month.name == "{}"',
'expression': 'expression.name == "{}"',
'speaker': 'speaker == "{}"',
# 'background':'background.name == "{}"',
}
if request_dict['cancel'] != 'on':
for k in filter_pattern_dict:
if request_dict.get(k):
filter_dict[request_dict.get(
k)] = filter_pattern_dict[k].format(request_dict.get(k))
query_para_dict[k] = request_dict.get(k)
# グルンガのフィルタクエリ作成と絞り込みメッセージ作成
filter_msg = ""
if filter_dict:
groonga_query_dict['filter'] = ' && '.join(filter_dict.values())
filter_msg = " ({}で絞り込み)".format('、'.join(filter_dict.keys()))
# ページ処理用のクエリ文字作成
query_string = urllib.parse.urlencode(query_para_dict)
# グルンガのページクエリ作成
if int(request_dict['page']) > 1:
groonga_query_dict['offset'] = 10 * (int(request_dict['page']) - 1)
# グルンガ実行
g = Groonga(host="groonga", protocol="gqtp", port=10043)
cmds = [("select", groonga_query_dict)]
for cmd, kwargs in cmds:
ret = g.call(cmd, **kwargs)
# グルンガ実行結果(ステータスコード0以外はエラー、件数が取得出来ない場合もエラー)
if ret.status != 0:
return render_template('_index.html',
message="内部でエラー({})が発生しました。".format(ret.status))
result = ret.body
try:
result_count = ret.body[0][0][0]
except:
return render_template('_index.html',
message="内部でエラー({})が発生しました。".format(ret.status))
# 共通系結果をセット
common_dict = {
'status': ret.status,
'elapsed': ret.elapsed,
'count': result_count,
'word': request_dict['word'],
}
# 結果をセット
records = []
key = [
'id', 'title', 'subtitle', 'episode', 'speaker', 'text', 'taps',
'background', 'type', 'expression', 'month', 'key'
]
for lst in result[0][2:]:
records.append(dict(zip(key, lst)))
# 結果ドリルダウンをセット
drilldown_pattern_dict = {
'writer': ['_order', 'writer', 'count'],
'type': ['_order', 'type', 'count'],
'month': ['_order', 'month', 'count'],
'expression': ['expression', 'count'],
'speaker': ['speaker', 'count'],
# 'background':['background','count'],
}
drilldown_records_dict = {}
for pkey, plist in drilldown_pattern_dict.items():
tmplist = []
for glst in result[1][pkey][2:]:
tmplist.append(dict(zip(plist, glst)))
drilldown_records_dict[pkey] = tmplist
# flask_paginate
pagination = Pagination(
page=request_dict['page'],
total=result_count,
per_page=PER_PAGE,
css_framework='bootstrap4',
href="search?page={}&" + query_string,
record_name="「{}」".format(request_dict['word'].strip()),
found=filter_msg, # use filter
display_msg=
"{record_name}で検索、<b>{total}</b> 件ヒット [<b>{start} - {end}</b> 件を表示]{found}"
)
return render_template(
'_search.html',
pagination=pagination, # flask_paginate
common_dict=common_dict,
records=records,
drilldown_records_dict=drilldown_records_dict,
query_para_dict=query_para_dict)
import sys
from poyonga import Groonga
query = "content:@%s" % sys.argv[1]
grn = Groonga()
ret = grn.call("select",
table="Files",
limit=1000,
sortby="-_score",
output_columns="_score,_key",
query=query)
for item in ret.items:
print("[score:%3d]%s" % (item["_score"], item["_key"]))
print("hit:", ret.hit_num)
# coding: utf-8
from poyonga import Groonga
def _call(g, cmd, **kwargs):
ret = g.call(cmd, **kwargs)
print "status:", ret.status
if cmd == 'select':
print "item:", len(ret.items)
for item in ret.items:
print item
print "=*=" * 30
g = Groonga()
_call(g,
"select",
table="Users",
match_columns="name,location_str,description",
query="東京",
output_columns="_key,name")
_call(g,
"select",
table="Users",
filter="geo_in_circle(location,\"128484216x502919856\",5000)",
output_columns="_key,name")
_call(g,
"select",
table="Users",
query="follower:@tasukuchan",
from poyonga import Groonga
g = Groonga(protocol="gqtp")
cmds = [
("status", {}),
("stat", {}), # invalid
# ("log_level", {"level": "warning"}),
# ("table_create", {"name": "Site", "flags": "TABLE_HASH_KEY"}),
("table_list", {}),
("select", {
"table": "Site"
}),
]
for cmd, kwargs in cmds:
ret = g.call(cmd, **kwargs)
print(ret.status)
print(ret.body)
print("*" * 40)
from poyonga import Groonga
g = Groonga(protocol="gqtp")
cmds = [("status", {}),
("stat", {}), # invalid
#("log_level", {"level": "warning"}),
#("table_create", {"name": "Site", "flags": "TABLE_HASH_KEY"}),
("table_list", {}),
("select", {"table": "Site"})
]
for cmd, kwargs in cmds:
ret = g.call(cmd, **kwargs)
print(ret.status)
print(ret.body)
print("*" * 40)
def setUp(self):
self.g = Groonga(protocol='gqtp')
from poyonga import Groonga
grn = Groonga()
ret = grn.call("table_create",
name="Files",
flags="TABLE_HASH_KEY",
key_type="ShortText")
print ret.status
ret = grn.call("table_create",
name="Bigram",
key_type="ShortText",
flags="TABLE_PAT_KEY|KEY_NORMALIZE",
default_tokenizer="TokenBigramSplitSymbolAlphaDigit")
print ret.status
ret = grn.call("column_create",
table="Files",
name="ext",
flags="COLUMN_SCALAR",
type="ShortText")
print ret.status
ret = grn.call("column_create",
table="Files",
name="name",
flags="COLUMN_SCALAR",
type="ShortText")
print ret.status
ret = grn.call("column_create",
table="Files",
name="content",
flags="COLUMN_SCALAR",
def setUp(self):
self.g = Groonga()
