class DbKyoto(object):
def __init__(self, db_file):
self.db = DB()
self.db_file = db_file
if not self.db.open(db_file, DB.OWRITER | DB.OCREATE):
print >> sys.stderr, 'open error: ' + str(self.db.error())
def set(self, txt, po_id):
feature_list = feature_md5(txt)
for feature in feature_list:
key = feature
entry = self.get(key)
if not entry:
val = array('L', [po_id])
if not self.db.set(key, val.tostring()):
print >> sys.stderr, 'open error: ' + str(self.db.error())
else:
val = array('L')
val.fromstring(entry)
if po_id not in val:
val.append(po_id)
self.db.set(key, val.tostring())
return val
def get(self, key):
po_id = self.db.get(key)
result = array('L')
if po_id:
result.fromstring(po_id)
return result
class DbKyoto(object):
def __init__(self, db_file):
self.db = DB()
self.db_file = db_file
if not self.db.open(db_file, DB.OWRITER | DB.OCREATE):
print >> sys.stderr, 'open error: ' + str(self.db.error())
def set(self, txt, po_id):
feature_list = feature_md5(txt)
for feature in feature_list:
key = feature
entry = self.get(key)
if not entry:
val = array('L', [po_id])
if not self.db.set(key, val.tostring()):
print >> sys.stderr, 'open error: ' + str(self.db.error())
else:
val = array('L')
val.fromstring(entry)
if po_id not in val:
val.append(po_id)
self.db.set(key, val.tostring())
return val
def get(self, key):
po_id = self.db.get(key)
result = array('L')
if po_id:
result.fromstring(po_id)
return result
def insert_item(url, person, db_file, submitted_title=''):
mimetype = "application/json"
db = DB()
if not db.open("{0}".format(db_file), DB.OWRITER | DB.OCREATE):
response = {}
response[
'What happened?'] = "Couldn't open the damn database. Error: {0}".format(
db.error())
return Response(dumps(response), mimetype=mimetype)
if is_url_in_db(db, url):
return Response('{"What happened?": "Someone '\
'tried to submit a duplicate URL."}',
mimetype=mimetype)
title = url
summary = "~?~"
try:
thing = urlopen(url, timeout=10)
soup = BeautifulSoup(thing)
title = soup.title.string
# Do some dumb summarizing if we can
def concat(a, v):
return a + " " + v.strip()
visible_stuff = filter(visible, soup.findAll(text=True))
summary = reduce(concat, visible_stuff, "")[:900] + "..."
except:
pass
#return Response('{"What happened?": '\
# 'I dunno bs4 messed up somehow."}',
# mimetype=mimetype)
created_at = int(mktime(datetime.now().utctimetuple()))
is_image = url.lower().endswith(("jpg", "jpeg", "gif", "png"))
thumbnail = gen_thumbnail_for_url(url, str(created_at))
record = {
"created_at": created_at,
"title": title,
"url": url,
"person": person,
"summary": summary,
"person_color": PERSON_COLORS[random.randint(0,
len(PERSON_COLORS) - 1)],
"is_image": is_image,
"thumbnail": thumbnail,
"comment": submitted_title
}
db.set(created_at, dumps(record))
db.close()
return Response('{"What happened?": "MUDADA"}', mimetype=mimetype)
def insert_item(url, person, db_file, submitted_title=''):
mimetype = "application/json"
db = DB()
if not db.open("{0}".format(db_file),
DB.OWRITER | DB.OCREATE):
response = {}
response['What happened?'] = "Couldn't open the damn database. Error: {0}".format(db.error())
return Response(dumps(response), mimetype=mimetype)
if is_url_in_db(db, url):
return Response('{"What happened?": "Someone '\
'tried to submit a duplicate URL."}',
mimetype=mimetype)
title = url
summary = "~?~"
try:
thing = urlopen(url, timeout=10)
soup = BeautifulSoup(thing)
title = ''
if len(submitted_title) > 0:
title = submitted_title
else:
title = soup.title.string
# Do some dumb summarizing if we can
func = lambda a,v: a + " " + v.strip()
visible_stuff = filter(visible, soup.findAll(text=True))
summary = reduce(func, visible_stuff, "")[:900] + "..."
except:
pass
#return Response('{"What happened?": '\
# 'I dunno bs4 messed up somehow."}',
# mimetype=mimetype)
created_at = int(mktime(datetime.now().utctimetuple()))
is_image = url.lower().endswith(("jpg", "jpeg", "gif", "png"))
thumbnail = gen_thumbnail_for_url(url, str(created_at))
record = {
"created_at": created_at,
"title": title,
"url": url,
"person": person,
"summary": summary,
"person_color": PERSON_COLORS[random.randint(0, len(PERSON_COLORS)-1)],
"is_image": is_image,
"thumbnail": thumbnail
}
db.set(created_at, dumps(record))
db.close()
return Response('{"What happened?": "MUDADA"}',
mimetype=mimetype)
class DbKyoto(object):
def __init__(self, db_file):
#from train import TAG2ID, WORD2ID#, BAYES_RANK
#self.ider = WORD2ID
self.db = DB()
self.db_file = db_file
print path.join(KYOTO_DB_PATH,self.db_file)
if not self.db.open(path.join(KYOTO_DB_PATH,self.db_file), DB.OWRITER | DB.OCREATE):
print >>sys.stderr, "open error: " + str(self.db.error())
def set(self,entry):
key = entry[0]
result_array = convert2array(entry[1]).tostring()
if not self.db.set(key,result_array):
print key
print result_array
print >>sys.stderr, "open error: " + str(self.db.error())
def get(self,key):
value = self.db.get(key)
if value:
result = array('L')
result.fromstring(value)
return convert2dict(result)
else:
#print >>sys.stderr, self.ider.get_word_by_id(key)
#print key
#print >>sys.stderr, "%s error: "%key + str(self.db.error())
pass
class DbKyoto(object):
def __init__(self, db_file):
#from train import TAG2ID, WORD2ID#, BAYES_RANK
#self.ider = WORD2ID
self.db = DB()
self.db_file = db_file
print path.join(KYOTO_DB_PATH, self.db_file)
if not self.db.open(path.join(KYOTO_DB_PATH, self.db_file),
DB.OWRITER | DB.OCREATE):
print >> sys.stderr, "open error: " + str(self.db.error())
def set(self, entry):
key = entry[0]
result_array = convert2array(entry[1]).tostring()
if not self.db.set(key, result_array):
print key
print result_array
print >> sys.stderr, "open error: " + str(self.db.error())
def get(self, key):
value = self.db.get(key)
if value:
result = array('L')
result.fromstring(value)
return convert2dict(result)
else:
#print >>sys.stderr, self.ider.get_word_by_id(key)
#print key
#print >>sys.stderr, "%s error: "%key + str(self.db.error())
pass
def decorated_function(*args, **kwargs):
# Debug
if not current_app.config['CACHE']:
return f(*args, **kwargs)
db = DB()
db.open("/tmp/page_cache.kch")
res = None
fancy = hash("{}{}{}".format(db_meta_info()['count'], request.url, f.func_name))
res = db.get(fancy)
if not res:
res = f(*args, **kwargs)
db.set(fancy, res)
db.close()
return res
def purge(domain, genid):
if request.remote_addr not in settings.ALLOW:
return text_response("Not permitted.\n", 403)
db = DB()
if not db.open(settings.GENID_DATABASE, DB.OWRITER | DB.OCREATE):
return text_response("Failed to purge: cannot open database.\n", 501)
set_ok = db.set(domain, genid)
db.close()
if not set_ok:
return text_response("Failed to purge: cannot set genid.\n", 501)
else:
return text_response("Purged <%s>\n" % (domain,))
#format datetime
def format_datetime(timestamp):
return time.strftime('%Y.%m.%d @ %H:%M',time.localtime(timestamp))
#创建数据库对象
db=DB()
# open the database
if not db.open("../data/db.kch", DB.OWRITER | DB.OCREATE):
print >>sys.stderr, "open error: " + str(db.error())
cid='1'
#build for marks form
if not db.set("mark:"+cid+":markid", "1") or\
not db.set("mark:"+cid+":userid", "1") or\
not db.set("mark:"+cid+":boardid", "1") or\
not db.set("mark:"+cid+":fileid", "1") or\
not db.set("mark:"+cid+":description", "mark的内容描述") or\
not db.set("mark:"+cid+":content", "哇。。阳光暖暖的，好惬意的。") or\
not db.set("mark:"+cid+":ups", "100") or\
not db.set("mark:"+cid+":downs", "10") or\
not db.set("mark:"+cid+":hits", "110") or\
not db.set("mark:"+cid+":order", "1") or\
not db.set("maks:"+cid+":createdata", int(time.time())) or\
not db.set("maks:"+cid+":commentcount", "8") or\
not db.set("mark:count", "3"):
print >>sys.stderr, "set error: " + str(db.error())
#build for comments form
class KyotoCabinetGraph(BaseGraph):
def __init__(self, path):
# create the database object
self._path = path
self._db = DB()
# open the database
if not self._db.open(path, DB.OREADER | DB.OWRITER | DB.OCREATE):
raise GrapheekDataKyotoCabinetInitFailureException(
str(self._db.error()))
super(KyotoCabinetGraph, self).__init__()
self._ensure_prepared()
self._closed = False
# Start method overriding :
def _db_close(self):
if not self._closed:
self._db.close()
def _transaction_begin(self):
self._db.begin_transaction()
return True
def _transaction_commit(self, txn):
self._db.end_transaction(True)
def _transaction_rollback(self, txn):
self._db.end_transaction(False)
def _has_key(self, key):
return self._db.check(key) >= 0
def _get(self, txn, key):
raw_data = self._db.get(key)
if raw_data is None:
return UNDEFINED # Not returning None, as None is a valid value
return msgpack.loads(raw_data, encoding='utf8')
def _bulk_get(self, txn, keys):
result = {}
key_raw_datas = self._db.get_bulk(keys)
for key, raw_data in list(key_raw_datas.items()):
if PYTHON2: # pragma : no cover
k = key
else: # pragma : no cover
k = str(key, encoding='utf8')
result[k] = msgpack.loads(raw_data, encoding='utf8')
return result
def _set(self, txn, key, value):
res = self._db.set(key, msgpack.dumps(value, encoding='utf8'))
if not (res): # pragma : no cover
raise GrapheekDataKyotoCabinetException(
'KyotoCabinet : error while saving')
return res
def _bulk_set(self, txn, updates):
dic = {}
for key, value in list(updates.items()):
dic[key] = msgpack.dumps(value, encoding='utf8')
res = self._db.set_bulk(dic)
if res == -1: # pragma : no cover
raise GrapheekDataKyotoCabinetException(
'KyotoCabinet : error while saving')
return res
def _remove(self, txn, key):
# Contrary to LocalMemoryGraph implementation, it is not needed to wrap
# key removal in try.. except because KyotoCabinet only send "False"
# when key does not exist
# Thus ... _removemethod is idempotent (cf LocalMemoryGraph _remove method comment)
self._db.remove(key)
def _bulk_remove(self, txn, keys):
res = self._db.remove_bulk(list(keys))
if res == -1: # pragma : no cover
raise GrapheekDataKyotoCabinetException(
'KyotoCabinet : error while saving')
return res
def _remove_prefix(self, txn, prefix):
keys = self._db.match_prefix(prefix)
self._db.remove_bulk(keys)
# overriding list methods
# looks like a bucket of hacks, and yes indeed it is :)
# btw, it REALLY improves performance if we compare to default implementation which,
# in the case of KyotoCabinet would involve msgpack deserialization followed by a serialization
def _init_lst(self, txn, key):
res = self._db.set(key, '')
if not (res): # pragma : no cover
raise GrapheekDataKyotoCabinetException(
'KyotoCabinet : error while saving')
return res
def _get_lst(self, txn, key):
value = self._db.get(key)
if value is None:
return UNDEFINED
# look _append_to_lst code below to understand why a split is done
# And why resulting list is sliced from 1
if PYTHON2: # pragma : no cover
return list(map(int, value.split('|')[1:]))
return list(map(
int,
str(value, encoding='utf8').split('|')[1:])) # pragma : no cover
def _set_lst(self, txn, key, values):
newval = '|'.join([str(value) for value in values])
res = self._db.set(key, '|' + newval)
if not (res): # pragma : no cover
raise GrapheekDataKyotoCabinetException(
'KyotoCabinet : error while saving')
return res
def _bulk_get_lst(self, txn, keys):
dic_values = self._db.get_bulk(keys)
results = []
for key in keys:
if PYTHON2: # pragma : no cover
values = dic_values.get(key, UNDEFINED)
else: # pragma : no cover
values = dic_values.get(bytes(key, encoding='utf8'), UNDEFINED)
if values == UNDEFINED:
results.append([])
else:
if PYTHON2: # pragma : no cover
results.append(list(map(int, values.split('|')[1:])))
else: # pragma : no cover
results.append(
list(
map(int,
str(values, encoding='utf8').split('|')[1:])))
return results
def _append_to_lst(self, txn, key, value):
self._db.append(key, '|' + str(value))
def _bulk_append_to_lst(self, txn, key, values):
newval = '|'.join([str(value) for value in values])
self._db.append(key, '|' + newval)
def _remove_from_lst(self, txn, key, value):
old = self._db.get(key)
if not PYTHON2: # pragma : no cover
old = str(old, encoding='utf8')
# Caution : we are only removing ONE occurence
# This is voluntary
# For instance, it lst contains neighbour node, we need to remove only one occurence
# cause current entity and neighbour node can be linked multiple time
new = old.replace('|%s' % value, '', 1)
if new == old:
raise ValueError("list.remove(x): x not in list")
res = self._db.set(key, new)
if not (res): # pragma : no cover
raise GrapheekDataKyotoCabinetException(
'KyotoCabinet : error while saving')
return res
def _bulk_remove_from_lst(self, txn, key, values):
assert (len(values))
old = self._db.get(key)
if PYTHON2: # pragma : no cover
new = old
else: # pragma : no cover
new = str(old, encoding='utf8')
for value in values:
new = new.replace('|%s' % value, '', 1)
if new == old: # pragma : no cover
raise ValueError("list.remove(x): x not in list")
res = self._db.set(key, new)
if not (res): # pragma : no cover
raise GrapheekDataKyotoCabinetException(
'KyotoCabinet : error while saving')
return res
