class DbKyoto(object):
def __init__(self, db_file):
#from train import TAG2ID, WORD2ID#, BAYES_RANK
#self.ider = WORD2ID
self.db = DB()
self.db_file = db_file
print path.join(KYOTO_DB_PATH, self.db_file)
if not self.db.open(path.join(KYOTO_DB_PATH, self.db_file),
DB.OWRITER | DB.OCREATE):
print >> sys.stderr, "open error: " + str(self.db.error())
def set(self, entry):
key = entry[0]
result_array = convert2array(entry[1]).tostring()
if not self.db.set(key, result_array):
print key
print result_array
print >> sys.stderr, "open error: " + str(self.db.error())
def get(self, key):
value = self.db.get(key)
if value:
result = array('L')
result.fromstring(value)
return convert2dict(result)
else:
#print >>sys.stderr, self.ider.get_word_by_id(key)
#print key
#print >>sys.stderr, "%s error: "%key + str(self.db.error())
pass
def aggregate_by_hour(db_file):
# Initialize the dict with each hour
hours = {key: 0 for key in range(0,24)}
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
cur = db.cursor()
cur.jump_back()
while True:
rec = cur.get(False)
if not rec:
break
loaded = loads(rec[1])
unix = float(loaded['created_at'])
time = datetime.fromtimestamp(unix)
hours[time.hour] = hours[time.hour] + 1
cur.step_back()
cur.disable()
db.close()
hours = [{'name': "{}:00".format(key), 'data': [hours[key]]} for key in hours]
return hours
class DbKyoto(object):
def __init__(self, db_file):
#from train import TAG2ID, WORD2ID#, BAYES_RANK
#self.ider = WORD2ID
self.db = DB()
self.db_file = db_file
print path.join(KYOTO_DB_PATH,self.db_file)
if not self.db.open(path.join(KYOTO_DB_PATH,self.db_file), DB.OWRITER | DB.OCREATE):
print >>sys.stderr, "open error: " + str(self.db.error())
def set(self,entry):
key = entry[0]
result_array = convert2array(entry[1]).tostring()
if not self.db.set(key,result_array):
print key
print result_array
print >>sys.stderr, "open error: " + str(self.db.error())
def get(self,key):
value = self.db.get(key)
if value:
result = array('L')
result.fromstring(value)
return convert2dict(result)
else:
#print >>sys.stderr, self.ider.get_word_by_id(key)
#print key
#print >>sys.stderr, "%s error: "%key + str(self.db.error())
pass
def get_items(item_filter, db_file, page=0):
item_iter = 0
items = []
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
cur = db.cursor()
cur.jump_back()
while len(items) < FILTER_MAX:
rec = cur.get(False)
if not rec:
break
if item_iter != (FILTER_MAX * page):
if item_filter(rec):
item_iter = item_iter + 1
cur.step_back()
continue
if item_filter(rec):
items.append(rec)
cur.step_back()
cur.disable()
db.close()
sorted_items = sorted(items, key=get_key, reverse=True)
sorted_items_for_viewing = [loads(item[1]) for item in sorted_items]
for item in sorted_items_for_viewing:
if item['title'] is None or item['title'] == "":
item['title'] = item['url']
return sorted_items_for_viewing
def get_post_num(post_num, db_file):
item = None
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
cur = db.cursor()
cur.jump()
i = 0
while True:
rec = cur.get(False)
if not rec:
break
if i == post_num:
item = rec
cur.step()
i = i + 1
cur.disable()
db.close()
if item is not None:
return loads(item[1])
return dict()
def startup(self): ''' Open the index. ''' # pylint: disable=import-error,import-outside-toplevel from kyotocabinet import DB self._kyoto = DB() self._kyoto.open(self.path, DB.OWRITER | DB.OCREATE)
def init_write(self):
self.mode = "write"
if self.ext == ".csv":
self._data_file = open(self.filename, "wb")
self._writer = csv.writer(self._data_file)
if self.headers:
self._writer.writerow(self.headers)
elif self.ext == ".json":
self._storage = {}
elif self.ext == ".kch":
from kyotocabinet import DB
self._storage = DB()
if not self._storage.open(self.filename, DB.OWRITER | DB.OCREATE):
msg = "Error opening kyotocabinet db: %s" % (
self._storage.error())
raise dexy.exceptions.UserFeedback(msg)
elif self.ext == ".sqlite3":
self.init_write_sqlite3()
else:
raise dexy.exceptions.UserFeedback("unsupported extension %s" %
self.ext)
def open_db(self, name):
db = DB()
db.open(
join(self.path, "%s.kch"%name),
DB.OWRITER | DB.OCREATE
)
self.opendb.append(db)
return db
def open(self, filename):
db = DB()
# self.db=DB(DB.GEXCEPTIONAL)
if not db.open(filename, DB.OWRITER | DB.OCREATE | DB.ONOLOCK):
raise IOError("open error: '" + str(self.db.error()) + "' on file:"
+ filename)
return db
def __init__(self, db_file):
#from train import TAG2ID, WORD2ID#, BAYES_RANK
#self.ider = WORD2ID
self.db = DB()
self.db_file = db_file
print path.join(KYOTO_DB_PATH, self.db_file)
if not self.db.open(path.join(KYOTO_DB_PATH, self.db_file),
DB.OWRITER | DB.OCREATE):
print >> sys.stderr, "open error: " + str(self.db.error())
def __init__(self, dbpath, readonly=False):
self.readonly = readonly
mode = KDB.OREADER if readonly else KDB.OREADER | KDB.OWRITER | KDB.OCREATE
db = KDB()
if not db.open(dbpath, mode):
raise IOError("kyotocabinet.DB().open(%s, %o): %s" %
(dbpath, mode, db.error()))
self.db = db
self.dbpath = dbpath
def top_things(db_file):
urls = {}
people = {}
graph = {}
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database. (Top things)"
cur = db.cursor()
cur.jump_back()
while True:
rec = cur.get(False)
if not rec:
break
loaded_rec = loads(rec[1])
split = get_domain(loaded_rec)
if urls.get(split, False) == False:
urls[split] = 1
else:
urls[split] = urls[split] + 1
person = loaded_rec['person']
if people.get(person, False) == False:
people[person] = 1
else:
people[person] = people[person] + 1
if split is not None and split is not "" and \
person is not None and person is not "":
# Build a crazy relational graph out of my nosql data
if graph.get(split, False) == False:
graph[split] = {"is_person": False, "data": [person], "linked_to_count": 1}
elif person not in graph[split]:
graph[split]["data"].append(person)
graph[split]["linked_to_count"] = graph[split]["linked_to_count"] + 1
if graph.get(person, False) == False:
graph[person] = {"is_person": True, "data": [split]}
elif split not in graph[person]:
graph[person]["data"].append(split)
cur.step_back()
cur.disable()
db.close()
def get_one(x):
return x[1]
return (sorted(urls.items(), key=get_one, reverse=True),
sorted(people.items(), key=get_one, reverse=True),
graph)
def get_post_by_date(key, db_file):
item = None
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
item = db.get(key)
db.close()
if item is not None:
return loads(item)
return dict()
def __init__(self, path):
# create the database object
self._path = path
self._db = DB()
# open the database
if not self._db.open(path, DB.OREADER | DB.OWRITER | DB.OCREATE):
raise GrapheekDataKyotoCabinetInitFailureException(
str(self._db.error()))
super(KyotoCabinetGraph, self).__init__()
self._ensure_prepared()
self._closed = False
def dbOpen(name):
db = DB()
if self.create:
# if not db.open(abspath(self.path) + '/' + name + ".kch",
# DB.OWRITER | DB.OCREATE | DB.OAUTOSYNC | DB.OAUTOTRAN):
if not db.open(abspath(self.path) + "/" + name + ".kch", DB.OWRITER | DB.OCREATE):
raise IOError("open error: " + str(db.error()))
return db
else:
# if not db.open(abspath(self.path) + '/' + name + ".kch",
# DB.OWRITER | DB.OAUTOSYNC | DB.OAUTOTRAN):
if not db.open(abspath(self.path) + "/" + name + ".kch", DB.OWRITER):
raise IOError("open error: " + str(db.error()))
return db
def main():
db = DB()
if not db.open("bayes.kch", DB.OWRITER | DB.OCREATE):
return
with open("word_tf.txt") as word_tf:
for line in word_tf:
line = line.strip()
word, bayes_list = loads(line)
print word
if bayes_list:
ar = array('I')
ar.fromlist(lineiter(bayes_list))
db[word] = ar.tostring()
def main():
db = DB()
if not db.open("bayes.kch", DB.OWRITER | DB.OCREATE):
return
with open("word_tf.txt") as word_tf:
for line in word_tf:
line = line.strip()
word, bayes_list = loads(line)
print word
if bayes_list:
ar = array('I')
ar.fromlist(lineiter(bayes_list))
db[word] = ar.tostring()
def __load_blast_data(blast):
# Connect to kyoto db
db = DB()
if not db.open("/opt/gene2accession/gene2accession.kch", DB.OREADER):
raise Exception("Could not load gene2accession.kch: " + str(db.error()))
hits = {}
gi_num = re.compile('gi\|([0-9]+)')
for line in blast:
split_line = line.split('\t')
# Important data
evalue = float(split_line[10])
gi_nums = gi_num.findall(split_line[12])
genome_ids = [db.get(x) for x in gi_nums if db.get(x) is not None]
# Thanks to Peter's parser, the gi list and org list are the same
# length (the first gi column is also the first gi in the "master" gi
# column)
for org in genome_ids:
if org in hits:
hits[org].append(evalue)
else:
hits[org] = [evalue]
db.close()
return hits
def dbOpen(name):
db = DB()
dbpathname = abspath(self.path) + '/' + name + ".kch"
if self.create:
# if not db.open(abspath(self.path) + '/' + name + ".kch",
# DB.OWRITER | DB.OCREATE | DB.OAUTOSYNC | DB.OAUTOTRAN):
if not db.open(dbpathname, DB.OWRITER | DB.OCREATE):
raise IOError("open error: %s %s" % (dbpathname, str(db.error()))) #pragma: NO COVER
return db
else:
# if not db.open(abspath(self.path) + '/' + name + ".kch",
# DB.OWRITER | DB.OAUTOSYNC | DB.OAUTOTRAN):
if not db.open(dbpathname, DB.OWRITER): #pragma: NO COVER
raise IOError("open error: %s %s" % (dbpathname, str(db.error()))) #pragma: NO COVER
return db
def purge(domain, genid):
if request.remote_addr not in settings.ALLOW:
return text_response("Not permitted.\n", 403)
db = DB()
if not db.open(settings.GENID_DATABASE, DB.OWRITER | DB.OCREATE):
return text_response("Failed to purge: cannot open database.\n", 501)
set_ok = db.set(domain, genid)
db.close()
if not set_ok:
return text_response("Failed to purge: cannot set genid.\n", 501)
else:
return text_response("Purged <%s>\n" % (domain,))
def __enter__( self ):
if self._mode == "r":
with open( self._fn, "rb" ) as f:
state = pickle_load( f );
self._len_c = state[ "c" ];
self._len_b = state[ "b" ];
self._len_x = state[ "x" ];
self._lenrow = self._len_c + self._len_b + self._len_x;
self._ic = state[ "ic" ];
self._icbp = state[ "icbp" ];
if self._mode == "w":
with NamedTemporaryFile() as tmpfn:
self._kdbfn = tmpfn.name + '.kch';
self._kdb = KDB();
try:
assert self._kdb.open( self._kdbfn, KDB.OWRITER | KDB.OCREATE );
except:
print( str( self._kdb.error() ) );
raise;
with TemporaryDirectory() as tmpdirname:
self._ldbdn = tmpdirname;
self._ldb = LDB( self._ldbdn, create_if_missing=True );
return self;
def init_write(self):
self.mode = "write"
if self.ext == ".csv":
self._data_file = open(self.filename, "wb")
self._writer = csv.writer(self._data_file)
if self.headers:
self._writer.writerow(self.headers)
elif self.ext == ".json":
self._storage = {}
elif self.ext == ".kch":
from kyotocabinet import DB
self._storage = DB()
if not self._storage.open(self.filename, DB.OWRITER | DB.OCREATE):
msg = "Error opening kyotocabinet db: %s" % (self._storage.error())
raise dexy.commands.UserFeedback(msg)
elif self.ext == ".sqlite3":
self.init_write_sqlite3()
else:
raise dexy.commands.UserFeedback("unsupported extension %s" % self.ext)
def insert_item(url, person, db_file, submitted_title=''):
mimetype = "application/json"
db = DB()
if not db.open("{0}".format(db_file), DB.OWRITER | DB.OCREATE):
response = {}
response[
'What happened?'] = "Couldn't open the damn database. Error: {0}".format(
db.error())
return Response(dumps(response), mimetype=mimetype)
if is_url_in_db(db, url):
return Response('{"What happened?": "Someone '\
'tried to submit a duplicate URL."}',
mimetype=mimetype)
title = url
summary = "~?~"
try:
thing = urlopen(url, timeout=10)
soup = BeautifulSoup(thing)
title = soup.title.string
# Do some dumb summarizing if we can
def concat(a, v):
return a + " " + v.strip()
visible_stuff = filter(visible, soup.findAll(text=True))
summary = reduce(concat, visible_stuff, "")[:900] + "..."
except:
pass
#return Response('{"What happened?": '\
# 'I dunno bs4 messed up somehow."}',
# mimetype=mimetype)
created_at = int(mktime(datetime.now().utctimetuple()))
is_image = url.lower().endswith(("jpg", "jpeg", "gif", "png"))
thumbnail = gen_thumbnail_for_url(url, str(created_at))
record = {
"created_at": created_at,
"title": title,
"url": url,
"person": person,
"summary": summary,
"person_color": PERSON_COLORS[random.randint(0,
len(PERSON_COLORS) - 1)],
"is_image": is_image,
"thumbnail": thumbnail,
"comment": submitted_title
}
db.set(created_at, dumps(record))
db.close()
return Response('{"What happened?": "MUDADA"}', mimetype=mimetype)
def __enter__( self ):
self._cdata = DB();
self._ddata = DB();
try:
if self._mode == "r":
assert self._cdata.open( self._fn_cdata, DB.OREADER );
elif self._mode == "w":
if isfile( self._fn_cdata ):
remove( self._fn_cdata );
assert self._cdata.open( self._fn_cdata, DB.OWRITER | DB.OCREATE );
else:
assert False;
except:
if self._cdata is not None:
print( str( self._cdata.error() ) );
raise;
try:
if self._mode == "r":
assert self._ddata.open( self._fn_ddata, DB.OREADER );
elif self._mode == "w":
if isfile( self._fn_ddata ):
remove( self._fn_ddata );
assert self._ddata.open( self._fn_ddata, DB.OWRITER | DB.OCREATE );
else:
assert False;
except:
if self._ddata is not None:
print( str( self._ddata.error() ) );
raise;
if self._mode == "r":
with open( self._fn_meta, 'rb' ) as f:
r = pickle_load( f );
self._len_c = r[ "c" ];
self._len_b = r[ "b" ];
self._len_x = r[ "x" ];
self._co = r[ "co" ];
with open( self._fn_icov, 'rb' ) as f:
self._icov = pickle_load( f );
return self;
def __init__(self, db_file):
#from train import TAG2ID, WORD2ID#, BAYES_RANK
#self.ider = WORD2ID
self.db = DB()
self.db_file = db_file
print path.join(KYOTO_DB_PATH,self.db_file)
if not self.db.open(path.join(KYOTO_DB_PATH,self.db_file), DB.OWRITER | DB.OCREATE):
print >>sys.stderr, "open error: " + str(self.db.error())
class DbKyoto(object):
def __init__(self, db_file):
self.db = DB()
self.db_file = db_file
if not self.db.open(db_file, DB.OWRITER | DB.OCREATE):
print >> sys.stderr, 'open error: ' + str(self.db.error())
def set(self, txt, po_id):
feature_list = feature_md5(txt)
for feature in feature_list:
key = feature
entry = self.get(key)
if not entry:
val = array('L', [po_id])
if not self.db.set(key, val.tostring()):
print >> sys.stderr, 'open error: ' + str(self.db.error())
else:
val = array('L')
val.fromstring(entry)
if po_id not in val:
val.append(po_id)
self.db.set(key, val.tostring())
return val
def get(self, key):
po_id = self.db.get(key)
result = array('L')
if po_id:
result.fromstring(po_id)
return result
class DbKyoto(object):
def __init__(self, db_file):
self.db = DB()
self.db_file = db_file
if not self.db.open(db_file, DB.OWRITER | DB.OCREATE):
print >> sys.stderr, 'open error: ' + str(self.db.error())
def set(self, txt, po_id):
feature_list = feature_md5(txt)
for feature in feature_list:
key = feature
entry = self.get(key)
if not entry:
val = array('L', [po_id])
if not self.db.set(key, val.tostring()):
print >> sys.stderr, 'open error: ' + str(self.db.error())
else:
val = array('L')
val.fromstring(entry)
if po_id not in val:
val.append(po_id)
self.db.set(key, val.tostring())
return val
def get(self, key):
po_id = self.db.get(key)
result = array('L')
if po_id:
result.fromstring(po_id)
return result
def __init__(self, dir, subdirs=[]):
dir += "_zlib"
if dir not in dbList:
dbList[dir] = DB()
dbList[dir].open(
dir + ".kch#ops=c#log=" + dir +
".log#logkinds=debu#zcomp=zlib", DB.OWRITER | DB.OCREATE)
self.db = dbList[dir]
self.key = "/".join(subdirs)
def get_items_last_X_days(db_file, X, munge=True):
dates = {}
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
X_days_ago = datetime.now() - timedelta(days=X)
cur = db.cursor()
cur.jump_back()
while True:
rec = cur.get(False)
if not rec:
break
loaded = loads(rec[1])
unix = float(loaded['created_at'])
time = datetime.fromtimestamp(unix)
if time > X_days_ago:
if munge:
date_obj = date(year=time.year, month=time.month, day=time.day)
else:
date_obj = time
# Javascript expects Date.UTC to spit out dates of a certain
# length.
day_unix = int(mktime(date_obj.timetuple()))*1000
if dates.get(day_unix, False) == False:
dates[day_unix] = {loaded["person"]: 1}
else:
relevant_dict = dates[day_unix]
if relevant_dict.get(loaded["person"], False) == False:
relevant_dict[loaded["person"]] = 1
else:
relevant_dict[loaded["person"]] = relevant_dict[loaded["person"]] + 1
else:
break;
cur.step_back()
cur.disable()
db.close()
return dates
def get_last_items(db_file, pages=1):
items = []
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
cur = db.cursor()
cur.jump_back()
while len(items) < (pages * FILTER_MAX):
rec = cur.get(False)
if not rec:
break
items.append(rec)
cur.step_back()
cur.disable()
db.close()
return items
def init_read(self):
self.mode = "read"
if self.ext == ".csv":
self._file = open(self.filename, "rb")
elif self.ext == ".json":
with open(self.filename, "rb") as f:
self._storage = json.load(f)
elif self.ext == ".kch":
from kyotocabinet import DB
self._storage = DB()
self._storage.open(self.filename, DB.OREADER)
elif self.ext == ".sqlite3":
import sqlite3
self._storage = sqlite3.connect(self.filename)
self._cursor = self._storage.cursor()
else:
raise dexy.exceptions.UserFeedback("unsupported extension %s" %
self.ext)
def get_all_items(db_file):
items = []
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
cur = db.cursor()
cur.jump()
while True:
rec = cur.get(False)
if not rec:
break
items.append(rec)
cur.step()
cur.disable()
db.close()
sorted_items_for_viewing = [loads(item[1]) for item in items]
return sorted_items_for_viewing
def decorated_function(*args, **kwargs):
# Debug
if not current_app.config['CACHE']:
return f(*args, **kwargs)
db = DB()
db.open("/tmp/page_cache.kch")
res = None
fancy = hash("{}{}{}".format(db_meta_info()['count'], request.url, f.func_name))
res = db.get(fancy)
if not res:
res = f(*args, **kwargs)
db.set(fancy, res)
db.close()
return res
def get_items(item_filter, db_file, page=0):
item_iter = 0
items = []
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
cur = db.cursor()
cur.jump_back()
while len(items) < FILTER_MAX:
rec = cur.get(False)
if not rec:
break
if item_iter != (FILTER_MAX * page):
if item_filter(rec):
item_iter = item_iter + 1
cur.step_back()
continue
if item_filter(rec):
items.append(rec)
cur.step_back()
cur.disable()
db.close()
sorted_items = sorted(items, key=get_key, reverse=True)
sorted_items_for_viewing = [loads(item[1]) for item in sorted_items]
for item in sorted_items_for_viewing:
if item['title'] is None or item['title'] == "":
item['title'] = item['url']
return sorted_items_for_viewing
def get_post_num(post_num, db_file):
item = None
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
cur = db.cursor()
cur.jump()
i = 0
while True:
rec = cur.get(False)
if not rec:
break
if i == post_num:
item = rec
cur.step()
i = i + 1
cur.disable()
db.close()
if item is not None:
return loads(item[1])
return dict()
def aggregate_by_hour(db_file):
# Initialize the dict with each hour
hours = {key: 0 for key in range(0, 24)}
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
cur = db.cursor()
cur.jump_back()
while True:
rec = cur.get(False)
if not rec:
break
loaded = loads(rec[1])
unix = float(loaded['created_at'])
time = datetime.fromtimestamp(unix)
hours[time.hour] = hours[time.hour] + 1
cur.step_back()
cur.disable()
db.close()
hours = [{
'name': "{}:00".format(key),
'data': [hours[key]]
} for key in hours]
return hours
def db_meta_info():
meta = {}
db = DB()
db_file = current_app.config['DB_FILE']
if not db.open("{0}".format(db_file), DB.OREADER):
print "Could not open database (meta info)."
meta["size"] = db.size()
meta["count"] = db.count()
db.close()
return meta
def gen_thumbnails(db_file):
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OWRITER):
sys.exit(1)
cur = db.cursor()
cur.jump_back()
while True:
rec = cur.get(False)
if not rec:
break
loaded = loads(rec[1])
is_image = loaded["url"].lower().endswith(("jpg", "jpeg", "gif", "png"))
if is_image:
print "Thumbnailing {}".format(loaded["url"])
loaded["is_image"] = True
try:
thumbnail = gen_thumbnail_for_url(loaded["url"], rec[0])
except IOError as e:
print "IOError: {}".format(e)
print "Save result: {}".format(cur.set_value(dumps(loaded)))
cur.step_back()
continue
if thumbnail:
loaded["thumbnail"] = thumbnail
print "Thumbnailed {}".format(loaded["url"])
print "Save result: {}".format(cur.set_value(dumps(loaded)))
cur.step_back()
cur.disable()
db.close()
return True
def __init__(self, path, truncate=False):
"""
Open a new connection to a database using the Kyoto Cabinet engine.
Args:
* path (str): Path to database.
Kwargs:
* truncate (bool, False): If database should be truncated before opening.
"""
self.db = DB()
self.batchsize = 1000
self.batch = {}
dbparams = '.kct#apow=0#bnum=10000000#msiz=' + str(2 << 30)
if truncate:
result = self.db.open(path + dbparams,
DB.OWRITER | DB.OCREATE | DB.OTRUNCATE)
else:
result = self.db.open(path + dbparams, DB.OWRITER)
if not result:
raise PathError('DNA outdb open error: %s ' % self.db.error())
exit(1)
def top_things(db_file):
urls = {}
people = {}
graph = {}
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database. (Top things)"
cur = db.cursor()
cur.jump_back()
while True:
rec = cur.get(False)
if not rec:
break
loaded_rec = loads(rec[1])
split = get_domain(loaded_rec)
if urls.get(split, False) == False:
urls[split] = 1
else:
urls[split] = urls[split] + 1
person = loaded_rec['person']
if people.get(person, False) == False:
people[person] = 1
else:
people[person] = people[person] + 1
if split is not None and split is not "" and \
person is not None and person is not "":
# Build a crazy relational graph out of my nosql data
if graph.get(split, False) == False:
graph[split] = {
"is_person": False,
"data": [person],
"linked_to_count": 1
}
elif person not in graph[split]:
graph[split]["data"].append(person)
graph[split][
"linked_to_count"] = graph[split]["linked_to_count"] + 1
if graph.get(person, False) == False:
graph[person] = {"is_person": True, "data": [split]}
elif split not in graph[person]:
graph[person]["data"].append(split)
cur.step_back()
cur.disable()
db.close()
def get_one(x):
return x[1]
return (sorted(urls.items(), key=get_one, reverse=True),
sorted(people.items(), key=get_one, reverse=True), graph)
def get_items_on_page(page, db_file):
item_iter = 0
items = []
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
cur = db.cursor()
cur.jump_back()
while len(items) < FILTER_MAX:
rec = cur.get(False)
if not rec:
break
if item_iter >= (FILTER_MAX * page):
items.append(rec)
item_iter = item_iter + 1
cur.step_back()
cur.disable()
db.close()
return items
def insert_item(url, person, db_file, submitted_title=''):
mimetype = "application/json"
db = DB()
if not db.open("{0}".format(db_file),
DB.OWRITER | DB.OCREATE):
response = {}
response['What happened?'] = "Couldn't open the damn database. Error: {0}".format(db.error())
return Response(dumps(response), mimetype=mimetype)
if is_url_in_db(db, url):
return Response('{"What happened?": "Someone '\
'tried to submit a duplicate URL."}',
mimetype=mimetype)
title = url
summary = "~?~"
try:
thing = urlopen(url, timeout=10)
soup = BeautifulSoup(thing)
title = ''
if len(submitted_title) > 0:
title = submitted_title
else:
title = soup.title.string
# Do some dumb summarizing if we can
func = lambda a,v: a + " " + v.strip()
visible_stuff = filter(visible, soup.findAll(text=True))
summary = reduce(func, visible_stuff, "")[:900] + "..."
except:
pass
#return Response('{"What happened?": '\
# 'I dunno bs4 messed up somehow."}',
# mimetype=mimetype)
created_at = int(mktime(datetime.now().utctimetuple()))
is_image = url.lower().endswith(("jpg", "jpeg", "gif", "png"))
thumbnail = gen_thumbnail_for_url(url, str(created_at))
record = {
"created_at": created_at,
"title": title,
"url": url,
"person": person,
"summary": summary,
"person_color": PERSON_COLORS[random.randint(0, len(PERSON_COLORS)-1)],
"is_image": is_image,
"thumbnail": thumbnail
}
db.set(created_at, dumps(record))
db.close()
return Response('{"What happened?": "MUDADA"}',
mimetype=mimetype)
def get_post_by_date(key, db_file):
item = None
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
item = db.get(key)
db.close()
if item is not None:
return loads(item)
return dict()
def db_meta_info():
meta = {}
db = DB()
db_file = current_app.config['DB_FILE']
if not db.open("{0}".format(db_file), DB.OREADER):
print "Could not open database (meta info)."
meta["size"] = db.size()
meta["count"] = db.count()
db.close()
return meta
class KDB:
def __init__(self, path, truncate=False):
"""
Open a new connection to a database using the Kyoto Cabinet engine.
Args:
* path (str): Path to database.
Kwargs:
* truncate (bool, False): If database should be truncated before opening.
"""
self.db = DB()
self.batchsize = 1000
self.batch = {}
dbparams = '.kct#apow=0#bnum=10000000#msiz=' + str(2 << 30)
if truncate:
result = self.db.open(path + dbparams,
DB.OWRITER | DB.OCREATE | DB.OTRUNCATE)
else:
result = self.db.open(path + dbparams, DB.OWRITER)
if not result:
raise PathError('DNA outdb open error: %s ' % self.db.error())
exit(1)
def get(self, key):
"""
Retrieve the item with the given `key`.
"""
return self.db.get(key)
def put(self, key, val):
"""
Put `val` at `key`.
Note that disk writing is done in batches, so be sure to call `close` or `flush` to make sure that values are put into the store.
"""
self.batch[key] = val
if len(self.batch) >= self.batchsize:
self.flush()
def flush(self):
"""
Write `put` calls to database.
"""
self.db.set_bulk(self.batch)
self.batch = {}
def close(self):
"""
Flush the database and delete the connection to it.
"""
self.flush()
del self.db
def init_read(self):
self.mode = "read"
if self.ext == ".csv":
self._file = open(self.filename, "rb")
elif self.ext == ".json":
with open(self.filename, "rb") as f:
self._storage = json.load(f)
elif self.ext == ".kch":
from kyotocabinet import DB
self._storage = DB()
self._storage.open(self.filename, DB.OREADER)
elif self.ext == ".sqlite3":
import sqlite3
self._storage = sqlite3.connect(self.filename)
self._cursor = self._storage.cursor()
else:
raise dexy.exceptions.UserFeedback("unsupported extension %s" % self.ext)
def main():
db_file = argv[1]
username = argv[2]
if not db_file and not username:
print "Need db_file and username."
return -1
db = DB()
if not db.open("{0}".format(db_file), DB.OWRITER):
print "Could not open database."
return -1
all_keys = []
cur = db.cursor()
cur.jump()
while True:
rec = cur.get(False)
if not rec:
break
loaded = loads(rec[1])
if loaded["person"] == username:
all_keys.append(cur.get_key())
cur.step()
cur.disable()
print "Found {} records.".format(len(all_keys))
for key in all_keys:
print "Pending {}...".format(key)
if len(argv) > 3 and argv[3] == '--delete':
print "Removing {}...".format(key)
if not db.remove(key):
print "Could not remove key: {}".format(db.error())
db.close()
def dbOpen(name):
db = DB()
dbpathname = abspath(self.path) + '/' + name + ".kch"
if self.create:
# if not db.open(abspath(self.path) + '/' + name + ".kch",
# DB.OWRITER | DB.OCREATE | DB.OAUTOSYNC | DB.OAUTOTRAN):
if not db.open(dbpathname, DB.OWRITER | DB.OCREATE):
raise IOError("open error: %s %s" % (
dbpathname, str(db.error()))) # pragma: NO COVER
return db
else:
# if not db.open(abspath(self.path) + '/' + name + ".kch",
# DB.OWRITER | DB.OAUTOSYNC | DB.OAUTOTRAN):
if not db.open(dbpathname, DB.OWRITER): # pragma: NO COVER
raise IOError("open error: %s %s" % (
dbpathname, str(db.error()))) # pragma: NO COVER
return db
def get_items_last_X_days(db_file, X, munge=True):
dates = {}
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
X_days_ago = datetime.now() - timedelta(days=X)
cur = db.cursor()
cur.jump_back()
while True:
rec = cur.get(False)
if not rec:
break
loaded = loads(rec[1])
unix = float(loaded['created_at'])
time = datetime.fromtimestamp(unix)
if time > X_days_ago:
if munge:
date_obj = date(year=time.year, month=time.month, day=time.day)
else:
date_obj = time
# Javascript expects Date.UTC to spit out dates of a certain
# length.
day_unix = int(mktime(date_obj.timetuple())) * 1000
if dates.get(day_unix, False) == False:
dates[day_unix] = {loaded["person"]: 1}
else:
relevant_dict = dates[day_unix]
if relevant_dict.get(loaded["person"], False) == False:
relevant_dict[loaded["person"]] = 1
else:
relevant_dict[
loaded["person"]] = relevant_dict[loaded["person"]] + 1
else:
break
cur.step_back()
cur.disable()
db.close()
return dates
def get_last_items(db_file, pages=1):
items = []
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
cur = db.cursor()
cur.jump_back()
while len(items) < (pages * FILTER_MAX):
rec = cur.get(False)
if not rec:
break
items.append(rec)
cur.step_back()
cur.disable()
db.close()
return items
def main():
db_file = argv[1]
username = argv[2]
if not db_file and not username:
print "Need db_file and username."
return -1
db = DB()
if not db.open("{0}".format(db_file), DB.OWRITER):
print "Could not open database."
return -1
all_keys = []
cur = db.cursor()
cur.jump()
while True:
rec = cur.get(False)
if not rec:
break
loaded = loads(rec[1])
if loaded["person"] == username:
all_keys.append(cur.get_key())
cur.step()
cur.disable()
print "Found {} records.".format(len(all_keys))
for key in all_keys:
print "Pending {}...".format(key)
if len(argv) > 3 and argv[3] == '--delete':
print "Removing {}...".format(key)
if not db.remove(key):
print "Could not remove key: {}".format(db.error())
db.close()
def get_all_items(db_file):
items = []
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
cur = db.cursor()
cur.jump()
while True:
rec = cur.get(False)
if not rec:
break
items.append(rec)
cur.step()
cur.disable()
db.close()
sorted_items_for_viewing = [loads(item[1]) for item in items]
return sorted_items_for_viewing
def get_page_count(item_filter=lambda x: True):
count = 0
db = DB()
db_file = current_app.config['DB_FILE']
if not db.open("{0}".format(db_file),
DB.OREADER | DB.OWRITER | DB.OCREATE):
print "Could not open database (get_page_count). Error: {}".format(
db.error())
cur = db.cursor()
cur.jump_back()
while True:
rec = cur.get(False)
if not rec:
break
if item_filter(rec):
count = count + 1
cur.step_back()
cur.disable()
db.close()
return count / FILTER_MAX
def gen_thumbnails(db_file):
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OWRITER):
sys.exit(1)
cur = db.cursor()
cur.jump_back()
while True:
rec = cur.get(False)
if not rec:
break
loaded = loads(rec[1])
is_image = loaded["url"].lower().endswith(
("jpg", "jpeg", "gif", "png"))
if is_image:
print "Thumbnailing {}".format(loaded["url"])
loaded["is_image"] = True
try:
thumbnail = gen_thumbnail_for_url(loaded["url"], rec[0])
except IOError as e:
print "IOError: {}".format(e)
print "Save result: {}".format(cur.set_value(dumps(loaded)))
cur.step_back()
continue
if thumbnail:
loaded["thumbnail"] = thumbnail
print "Thumbnailed {}".format(loaded["url"])
print "Save result: {}".format(cur.set_value(dumps(loaded)))
cur.step_back()
cur.disable()
db.close()
return True
def get_page_count(item_filter = lambda x: True):
count = 0
db = DB()
db_file = current_app.config['DB_FILE']
if not db.open("{0}".format(db_file), DB.OREADER | DB.OWRITER | DB.OCREATE):
print "Could not open database (get_page_count). Error: {}".format(db.error())
cur = db.cursor()
cur.jump_back()
while True:
rec = cur.get(False)
if not rec:
break
if item_filter(rec):
count = count + 1
cur.step_back()
cur.disable()
db.close()
return count / FILTER_MAX
def get_items_on_page(page, db_file):
item_iter = 0
items = []
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
cur = db.cursor()
cur.jump_back()
while len(items) < FILTER_MAX:
rec = cur.get(False)
if not rec:
break
if item_iter >= (FILTER_MAX * page):
items.append(rec)
item_iter = item_iter + 1
cur.step_back()
cur.disable()
db.close()
return items
class FeatureSelector( Frontend ):
def __init__( self, fn, mode ):
Frontend.__init__( self, fn, mode );
self._kdbfn = None;
self._kdb = None;
self._ldbdn = None;
self._ldb = None;
self._len_c = None;
self._len_b = None;
self._len_x = None;
self._ic = None;
self._icbp = None;
self._needs_initialization = True;
self._core_dims = set();
self._satellite_dims = set();
self._removed_dims = set();
self._remove_c = set();
self._remove_b = set();
self._remove_x = set();
self.bypass_c = False;
self.bypass_b = False;
self.bypass_x = False;
def __enter__( self ):
if self._mode == "r":
with open( self._fn, "rb" ) as f:
state = pickle_load( f );
self._len_c = state[ "c" ];
self._len_b = state[ "b" ];
self._len_x = state[ "x" ];
self._lenrow = self._len_c + self._len_b + self._len_x;
self._ic = state[ "ic" ];
self._icbp = state[ "icbp" ];
if self._mode == "w":
with NamedTemporaryFile() as tmpfn:
self._kdbfn = tmpfn.name + '.kch';
self._kdb = KDB();
try:
assert self._kdb.open( self._kdbfn, KDB.OWRITER | KDB.OCREATE );
except:
print( str( self._kdb.error() ) );
raise;
with TemporaryDirectory() as tmpdirname:
self._ldbdn = tmpdirname;
self._ldb = LDB( self._ldbdn, create_if_missing=True );
return self;
def __exit__( self, exc_type, exc_value, traceback ):
assert Frontend.__exit__( self, exc_type, exc_value, traceback ) == False;
if self._ldb is not None:
sleep( 3.0 );
self._ldb.close()
if self._ldbdn is not None:
rmtree( self._ldbdn );
if self._kdb is not None:
try:
assert self._kdb.close();
except:
print( str( self._kdb.error() ) );
raise;
if self._kdbfn is not None:
remove( self._kdbfn );
def train( self, row ):
( y, c, b, x ) = row;
if self._len_c is None:
self._len_c = len(c);
assert self._len_c == len(c);
if self._len_b is None:
self._len_b = len(b);
assert self._len_b == len(b);
if self._len_x is None:
self._len_x = len(x);
assert self._len_x == len(x);
row = c + b + x;
if Frontend.train( self, row ):
return True;
keyfmt = '>IIIII';
for i in range( 0, self._lenrow ):
for j in range( 0, self._lenrow ):
if ( i >= j ) and ( not ( i == self._lenrow-1 ) ):
continue;
key = pack( keyfmt, i, j, y, row[i], row[j] );
try:
assert self._kdb.increment( key, 1, 0 );
except:
print( str(self._kdb.error()) );
raise;
def _stats( self, cnt_by_a, cnt_by_b, cnt_by_ab ):
h_a = 0.0;
h_b = 0.0;
h_ab = 0.0;
for ( val_a, cnt ) in cnt_by_a.items():
p = float(cnt) / float(self._rowcount);
if p > 0.0:
h_a -= p * log( p, 2.0 );
for ( val_b, cnt ) in cnt_by_b.items():
p = float(cnt) / float(self._rowcount);
if p > 0.0:
h_b -= p * log( p, 2.0 );
for( (val_a,val_b), cnt ) in cnt_by_ab.items():
p = float(cnt) / float(self._rowcount);
if p > 0.0:
h_ab -= p * log( p, 2.0 );
if h_a == 0.0:
return 1.0;
if h_b == 0.0:
return 1.0;
mi = h_a + h_b - h_ab;
return ( mi / min( h_a, h_b ), h_a, h_b, h_ab, mi );
def _get_info_content_by_dimension( self, i ):
keyfmt = '>IIIII';
valfmt = '>Q';
j = None;
cnt_by_a = {};
cnt_by_b = {};
cnt_by_ab = {};
total = 0;
with self._ldb.iterator() as it:
it.seek( pack( keyfmt, i,0,0,0,0 ) );
for ( key, val ) in it:
key = unpack( keyfmt, key );
val = unpack( valfmt, val )[ 0 ];
if not ( key[0] == i ):
break;
if j is None:
j = key[1];
if not ( key[1] == j ):
break;
# key[2] is the y-value
a = key[2];
# key[3] is the value for the i-th dimension
b = key[3];
cnt_by_ab[ (a,b) ] = cnt_by_ab.get( (a,b), 0 ) + val;
cnt_by_a[ a ] = cnt_by_a.get( a, 0 ) + val;
cnt_by_b[ b ] = cnt_by_b.get( b, 0 ) + val;
total += val;
try:
assert total == self._rowcount;
except:
print( i, j, total, self._rowcount );
raise;
return self._stats( cnt_by_a, cnt_by_b, cnt_by_ab );
def _get_info_content_by_pair( self, i, j ):
keyfmt = '>IIIII';
valfmt = '>Q';
cnt_by_a = {};
cnt_by_b = {};
cnt_by_ab = {};
total = 0;
with self._ldb.iterator() as it:
it.seek( pack( keyfmt, i,j,0,0,0 ) );
for ( key, val ) in it:
key = unpack( keyfmt, key );
val = unpack( valfmt, val )[ 0 ];
if not ( ( key[0] == i ) and ( key[1] == j ) ):
break;
# key[2] is the y-value, key[3] the i-th value for the i-th dim
a = ( key[2], key[3] );
# key[2] is the y-value, key[4] the i-th value for the j-th dim
b = ( key[2], key[4] );
assert (a,b) not in cnt_by_ab;
cnt_by_ab[ (a,b) ] = cnt_by_ab.get( (a,b), 0 ) + val;
cnt_by_a[ a ] = cnt_by_a.get( a, 0 ) + val;
cnt_by_b[ b ] = cnt_by_b.get( b, 0 ) + val;
total += val;
assert total == self._rowcount;
return self._stats( cnt_by_a, cnt_by_b, cnt_by_ab );
def _finalize( self ):
assert Frontend._finalize( self ) is None;
if False:
print( "unique combinations = ", self._kdb.count() );
keyfmt = '>IIIII';
valfmt = '>Q';
c = self._kdb.cursor();
c.jump();
gt2 = 0;
gt4 = 0;
gt8 = 0;
gt16 = 0;
gt32 = 0;
while True:
r = c.get( True );
if not r:
break;
self._ldb.put( r[0], r[1] );
key = unpack( keyfmt, r[0] );
val = unpack( valfmt, r[1] )[ 0 ];
if val > 2:
gt2 += 1;
if val > 4:
gt4 += 1;
if val > 8:
gt8 += 1;
if val > 16:
gt16 += 1;
if val > 32:
gt32 += 1;
if False:
print( gt2, gt4, gt8, gt16, gt32 );
self._ic = {};
for i in range( 0, self._lenrow ):
self._ic[ i ] = self._get_info_content_by_dimension( i );
self._icbp = {};
for i in range( 0, self._lenrow ):
for j in range( 0, self._lenrow ):
if i >= j:
continue;
self._icbp[ (i,j) ] = self._get_info_content_by_pair( i, j );
self._state \
= { "ic": self._ic,
"icbp": self._icbp,
"c": self._len_c,
"b": self._len_b,
"x": self._len_x };
def _fmt_dim( self, d_ ):
d = None;
if d_ < self._len_c:
d = "c" + str( d_ );
elif d_ < self._len_c + self._len_b:
d = "b" + str( d_ - self._len_c );
elif d_ < self._len_c + self._len_b + self._len_x:
d = "x" + str( d_ - self._len_c - self._len_b );
else:
assert False;
return "{:d}({:s})".format( d_, d );
def _init( self ):
self._needs_initialization = False;
if False:
for i in sorted( self._ic ):
(corr,h_a,h_b,h_ab,mi) = self._ic[ i ];
print(
"{:s} {:1.4f} {:1.4f} {:1.4f} {:1.4f} {:1.4f}"\
.format(
self._fmt_dim( i ),
corr,
h_a,
h_b,
h_ab,
mi
)
);
for (i,j) in sorted( self._icbp ):
(corr,h_a,h_b,h_ab,mi) = self._icbp[ (i,j) ];
print(
"{:s} {:s} {:1.4f} {:1.4f} {:1.4f} {:1.4f} {:1.4f}"\
.format(
self._fmt_dim( i ),
self._fmt_dim( j ),
corr,
h_a,
h_b,
h_ab,
mi
)
);
entropy \
= [ ( h_ab, i ) \
for ( i, (corr,h_a,h_b,h_ab,mi) ) in self._ic.items() ];
output_correlation \
= [ ( corr, i ) \
for ( i, (corr,h_a,h_b,h_ab,mi) ) in self._ic.items() ];
self._core_dims = set();
self._core_dims \
|= { i \
for ( h_ab, i ) \
in sorted( entropy, reverse=True )[ :5 ] };
self._core_dims \
|= { i \
for ( h_ab, i ) \
in sorted( output_correlation, reverse=True )[ :3 ] };
if True:
print(
"core = ",
" ".join([ self._fmt_dim(d) for d in self._core_dims ])
);
self._satellite_dims = set();
for core_dim in self._core_dims:
satellite_dim = None;
satellite_dim_c = None;
satellite_dim_stats = None;
for ( (i,j), (corr,h_a,h_b,h_ab,mi) ) in self._icbp.items():
if corr <= 0.5:
continue;
other_dim = None;
if i == core_dim:
other_dim = j;
elif j == core_dim:
other_dim = i;
else:
continue;
if ( satellite_dim_c is None ) or ( corr > satellite_dim_c ):
satellite_dim = other_dim;
satellite_dim_c = corr;
satellite_dim_stats = (corr,h_a,h_b,h_ab,mi);
if satellite_dim is not None:
self._satellite_dims.add( satellite_dim );
if False:
print(
'->',
self._fmt_dim(core_dim),
self._fmt_dim(satellite_dim)
);
print(
"{:1.4f} {:1.4f} {:1.4f} {:1.4f} {:1.4f}"\
.format( *(corr,h_a,h_b,h_ab,mi) )
);
if True:
print(
"satellite = ",
" ".join([ self._fmt_dim(d) for d in self._satellite_dims ])
);
self._removed_dims = set();
for i in self._ic:
if i not in self._core_dims and i not in self._satellite_dims:
self._removed_dims.add( i );
if True:
print(
"removed = ",
" ".join([ self._fmt_dim(d) for d in self._removed_dims ])
);
for d_ in self._removed_dims:
if d_ < self._len_c:
self._remove_c.add( d_ );
elif d_ < self._len_c + self._len_b:
self._remove_b.add( d_ - self._len_c );
elif d_ < self._len_c + self._len_b + self._len_x:
self._remove_x.add( d_ - self._len_c - self._len_b );
else:
assert False;
def apply_c( self, c ):
if self.bypass_c:
return c;
if self._needs_initialization:
self._init();
c_ = [];
for ( i, cval ) in enumerate( c ):
if not i in self._remove_c:
c_.append( cval );
return c_;
def apply_b( self, b ):
if self.bypass_b:
return b;
if self._needs_initialization:
self._init();
b_ = [];
for ( i, bval ) in enumerate( b ):
if not i in self._remove_b:
b_.append( bval );
return b_;
def apply_x( self, x ):
if self.bypass_x:
return x;
if self._needs_initialization:
self._init();
x_ = [];
for ( i, xval ) in enumerate( x ):
if not i in self._remove_x:
x_.append( xval );
return x_;
def __call__( self, row ):
if self._needs_initialization:
self._init();
( y, c, b, x ) = row;
y_ = y;
return \
( y_,
self.apply_c( c ),
self.apply_b( b ),
self.apply_x( x ) );
def get_user_stats(username, db_file):
item = {
"username": username,
"aliases": [],
"total_posts": 0,
"domains": {},
"first_post_date": None,
"first_post_date_unix": None,
"most_recent_post": None,
"most_recent_post_unix": 0,
"average_posts_per_hour": 0.0,
"average_posts_per_day": 0.0,
"average_posts_per_week": 0.0
}
db = DB()
if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE):
print "Could not open database."
cur = db.cursor()
cur.jump()
while True:
rec = cur.get(False)
if not rec:
break
loaded_rec = loads(rec[1])
if loaded_rec['person'] != username:
cur.step()
continue
# Looks like this is a post by the user we're looking for
split = get_domain(loaded_rec)
if item['domains'].get(split, False) == False:
item['domains'][split] = 1
else:
item['domains'][split] = item['domains'][split] + 1
if item['first_post_date_unix'] is None:
item['first_post_date_unix'] = loaded_rec['created_at']
if item['most_recent_post_unix'] < loaded_rec['created_at']:
item['most_recent_post_unix'] = loaded_rec['created_at']
item['total_posts'] = item['total_posts'] + 1
cur.step()
cur.disable()
db.close()
# Clean up everything
first_time = None
if item['first_post_date_unix'] is not None:
unix = float(item['first_post_date_unix'])
first_time = datetime.fromtimestamp(unix)
item['first_post_date'] = first_time.isoformat()
recent_time = None
if item['most_recent_post_unix'] is not None:
unix = float(item['most_recent_post_unix'])
recent_time = datetime.fromtimestamp(unix)
item['most_recent_post'] = recent_time.isoformat()
if first_time and recent_time:
delta = recent_time - first_time
item['user_age_days'] = delta.days
item['user_age_seconds'] = delta.total_seconds()
item['average_posts_per_hour'] = item['total_posts'] / (delta.total_seconds() / 60.0)
item['average_posts_per_day'] = item['total_posts'] / (delta.total_seconds() / 60.0 / 24.0)
item['average_posts_per_week'] = item['total_posts'] / (delta.total_seconds() / 60.0 / 24.0 / 7.0)
return item
