class DbKyoto(object): def __init__(self, db_file): #from train import TAG2ID, WORD2ID#, BAYES_RANK #self.ider = WORD2ID self.db = DB() self.db_file = db_file print path.join(KYOTO_DB_PATH, self.db_file) if not self.db.open(path.join(KYOTO_DB_PATH, self.db_file), DB.OWRITER | DB.OCREATE): print >> sys.stderr, "open error: " + str(self.db.error()) def set(self, entry): key = entry[0] result_array = convert2array(entry[1]).tostring() if not self.db.set(key, result_array): print key print result_array print >> sys.stderr, "open error: " + str(self.db.error()) def get(self, key): value = self.db.get(key) if value: result = array('L') result.fromstring(value) return convert2dict(result) else: #print >>sys.stderr, self.ider.get_word_by_id(key) #print key #print >>sys.stderr, "%s error: "%key + str(self.db.error()) pass
def aggregate_by_hour(db_file): # Initialize the dict with each hour hours = {key: 0 for key in range(0,24)} db = DB() if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE): print "Could not open database." cur = db.cursor() cur.jump_back() while True: rec = cur.get(False) if not rec: break loaded = loads(rec[1]) unix = float(loaded['created_at']) time = datetime.fromtimestamp(unix) hours[time.hour] = hours[time.hour] + 1 cur.step_back() cur.disable() db.close() hours = [{'name': "{}:00".format(key), 'data': [hours[key]]} for key in hours] return hours
class DbKyoto(object): def __init__(self, db_file): #from train import TAG2ID, WORD2ID#, BAYES_RANK #self.ider = WORD2ID self.db = DB() self.db_file = db_file print path.join(KYOTO_DB_PATH,self.db_file) if not self.db.open(path.join(KYOTO_DB_PATH,self.db_file), DB.OWRITER | DB.OCREATE): print >>sys.stderr, "open error: " + str(self.db.error()) def set(self,entry): key = entry[0] result_array = convert2array(entry[1]).tostring() if not self.db.set(key,result_array): print key print result_array print >>sys.stderr, "open error: " + str(self.db.error()) def get(self,key): value = self.db.get(key) if value: result = array('L') result.fromstring(value) return convert2dict(result) else: #print >>sys.stderr, self.ider.get_word_by_id(key) #print key #print >>sys.stderr, "%s error: "%key + str(self.db.error()) pass
def get_items(item_filter, db_file, page=0): item_iter = 0 items = [] db = DB() if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE): print "Could not open database." cur = db.cursor() cur.jump_back() while len(items) < FILTER_MAX: rec = cur.get(False) if not rec: break if item_iter != (FILTER_MAX * page): if item_filter(rec): item_iter = item_iter + 1 cur.step_back() continue if item_filter(rec): items.append(rec) cur.step_back() cur.disable() db.close() sorted_items = sorted(items, key=get_key, reverse=True) sorted_items_for_viewing = [loads(item[1]) for item in sorted_items] for item in sorted_items_for_viewing: if item['title'] is None or item['title'] == "": item['title'] = item['url'] return sorted_items_for_viewing
def get_post_num(post_num, db_file): item = None db = DB() if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE): print "Could not open database." cur = db.cursor() cur.jump() i = 0 while True: rec = cur.get(False) if not rec: break if i == post_num: item = rec cur.step() i = i + 1 cur.disable() db.close() if item is not None: return loads(item[1]) return dict()
def startup(self): ''' Open the index. ''' # pylint: disable=import-error,import-outside-toplevel from kyotocabinet import DB self._kyoto = DB() self._kyoto.open(self.path, DB.OWRITER | DB.OCREATE)
def init_write(self): self.mode = "write" if self.ext == ".csv": self._data_file = open(self.filename, "wb") self._writer = csv.writer(self._data_file) if self.headers: self._writer.writerow(self.headers) elif self.ext == ".json": self._storage = {} elif self.ext == ".kch": from kyotocabinet import DB self._storage = DB() if not self._storage.open(self.filename, DB.OWRITER | DB.OCREATE): msg = "Error opening kyotocabinet db: %s" % ( self._storage.error()) raise dexy.exceptions.UserFeedback(msg) elif self.ext == ".sqlite3": self.init_write_sqlite3() else: raise dexy.exceptions.UserFeedback("unsupported extension %s" % self.ext)
def open_db(self, name): db = DB() db.open( join(self.path, "%s.kch"%name), DB.OWRITER | DB.OCREATE ) self.opendb.append(db) return db
def open(self, filename): db = DB() # self.db=DB(DB.GEXCEPTIONAL) if not db.open(filename, DB.OWRITER | DB.OCREATE | DB.ONOLOCK): raise IOError("open error: '" + str(self.db.error()) + "' on file:" + filename) return db
def __init__(self, db_file): #from train import TAG2ID, WORD2ID#, BAYES_RANK #self.ider = WORD2ID self.db = DB() self.db_file = db_file print path.join(KYOTO_DB_PATH, self.db_file) if not self.db.open(path.join(KYOTO_DB_PATH, self.db_file), DB.OWRITER | DB.OCREATE): print >> sys.stderr, "open error: " + str(self.db.error())
def __init__(self, dbpath, readonly=False): self.readonly = readonly mode = KDB.OREADER if readonly else KDB.OREADER | KDB.OWRITER | KDB.OCREATE db = KDB() if not db.open(dbpath, mode): raise IOError("kyotocabinet.DB().open(%s, %o): %s" % (dbpath, mode, db.error())) self.db = db self.dbpath = dbpath
def top_things(db_file): urls = {} people = {} graph = {} db = DB() if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE): print "Could not open database. (Top things)" cur = db.cursor() cur.jump_back() while True: rec = cur.get(False) if not rec: break loaded_rec = loads(rec[1]) split = get_domain(loaded_rec) if urls.get(split, False) == False: urls[split] = 1 else: urls[split] = urls[split] + 1 person = loaded_rec['person'] if people.get(person, False) == False: people[person] = 1 else: people[person] = people[person] + 1 if split is not None and split is not "" and \ person is not None and person is not "": # Build a crazy relational graph out of my nosql data if graph.get(split, False) == False: graph[split] = {"is_person": False, "data": [person], "linked_to_count": 1} elif person not in graph[split]: graph[split]["data"].append(person) graph[split]["linked_to_count"] = graph[split]["linked_to_count"] + 1 if graph.get(person, False) == False: graph[person] = {"is_person": True, "data": [split]} elif split not in graph[person]: graph[person]["data"].append(split) cur.step_back() cur.disable() db.close() def get_one(x): return x[1] return (sorted(urls.items(), key=get_one, reverse=True), sorted(people.items(), key=get_one, reverse=True), graph)
def get_post_by_date(key, db_file): item = None db = DB() if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE): print "Could not open database." item = db.get(key) db.close() if item is not None: return loads(item) return dict()
def __init__(self, path): # create the database object self._path = path self._db = DB() # open the database if not self._db.open(path, DB.OREADER | DB.OWRITER | DB.OCREATE): raise GrapheekDataKyotoCabinetInitFailureException( str(self._db.error())) super(KyotoCabinetGraph, self).__init__() self._ensure_prepared() self._closed = False
def dbOpen(name): db = DB() if self.create: # if not db.open(abspath(self.path) + '/' + name + ".kch", # DB.OWRITER | DB.OCREATE | DB.OAUTOSYNC | DB.OAUTOTRAN): if not db.open(abspath(self.path) + "/" + name + ".kch", DB.OWRITER | DB.OCREATE): raise IOError("open error: " + str(db.error())) return db else: # if not db.open(abspath(self.path) + '/' + name + ".kch", # DB.OWRITER | DB.OAUTOSYNC | DB.OAUTOTRAN): if not db.open(abspath(self.path) + "/" + name + ".kch", DB.OWRITER): raise IOError("open error: " + str(db.error())) return db
def main(): db = DB() if not db.open("bayes.kch", DB.OWRITER | DB.OCREATE): return with open("word_tf.txt") as word_tf: for line in word_tf: line = line.strip() word, bayes_list = loads(line) print word if bayes_list: ar = array('I') ar.fromlist(lineiter(bayes_list)) db[word] = ar.tostring()
def __load_blast_data(blast): # Connect to kyoto db db = DB() if not db.open("/opt/gene2accession/gene2accession.kch", DB.OREADER): raise Exception("Could not load gene2accession.kch: " + str(db.error())) hits = {} gi_num = re.compile('gi\|([0-9]+)') for line in blast: split_line = line.split('\t') # Important data evalue = float(split_line[10]) gi_nums = gi_num.findall(split_line[12]) genome_ids = [db.get(x) for x in gi_nums if db.get(x) is not None] # Thanks to Peter's parser, the gi list and org list are the same # length (the first gi column is also the first gi in the "master" gi # column) for org in genome_ids: if org in hits: hits[org].append(evalue) else: hits[org] = [evalue] db.close() return hits
def dbOpen(name): db = DB() dbpathname = abspath(self.path) + '/' + name + ".kch" if self.create: # if not db.open(abspath(self.path) + '/' + name + ".kch", # DB.OWRITER | DB.OCREATE | DB.OAUTOSYNC | DB.OAUTOTRAN): if not db.open(dbpathname, DB.OWRITER | DB.OCREATE): raise IOError("open error: %s %s" % (dbpathname, str(db.error()))) #pragma: NO COVER return db else: # if not db.open(abspath(self.path) + '/' + name + ".kch", # DB.OWRITER | DB.OAUTOSYNC | DB.OAUTOTRAN): if not db.open(dbpathname, DB.OWRITER): #pragma: NO COVER raise IOError("open error: %s %s" % (dbpathname, str(db.error()))) #pragma: NO COVER return db
def purge(domain, genid): if request.remote_addr not in settings.ALLOW: return text_response("Not permitted.\n", 403) db = DB() if not db.open(settings.GENID_DATABASE, DB.OWRITER | DB.OCREATE): return text_response("Failed to purge: cannot open database.\n", 501) set_ok = db.set(domain, genid) db.close() if not set_ok: return text_response("Failed to purge: cannot set genid.\n", 501) else: return text_response("Purged <%s>\n" % (domain,))
def __enter__( self ): if self._mode == "r": with open( self._fn, "rb" ) as f: state = pickle_load( f ); self._len_c = state[ "c" ]; self._len_b = state[ "b" ]; self._len_x = state[ "x" ]; self._lenrow = self._len_c + self._len_b + self._len_x; self._ic = state[ "ic" ]; self._icbp = state[ "icbp" ]; if self._mode == "w": with NamedTemporaryFile() as tmpfn: self._kdbfn = tmpfn.name + '.kch'; self._kdb = KDB(); try: assert self._kdb.open( self._kdbfn, KDB.OWRITER | KDB.OCREATE ); except: print( str( self._kdb.error() ) ); raise; with TemporaryDirectory() as tmpdirname: self._ldbdn = tmpdirname; self._ldb = LDB( self._ldbdn, create_if_missing=True ); return self;
def init_write(self): self.mode = "write" if self.ext == ".csv": self._data_file = open(self.filename, "wb") self._writer = csv.writer(self._data_file) if self.headers: self._writer.writerow(self.headers) elif self.ext == ".json": self._storage = {} elif self.ext == ".kch": from kyotocabinet import DB self._storage = DB() if not self._storage.open(self.filename, DB.OWRITER | DB.OCREATE): msg = "Error opening kyotocabinet db: %s" % (self._storage.error()) raise dexy.commands.UserFeedback(msg) elif self.ext == ".sqlite3": self.init_write_sqlite3() else: raise dexy.commands.UserFeedback("unsupported extension %s" % self.ext)
def insert_item(url, person, db_file, submitted_title=''): mimetype = "application/json" db = DB() if not db.open("{0}".format(db_file), DB.OWRITER | DB.OCREATE): response = {} response[ 'What happened?'] = "Couldn't open the damn database. Error: {0}".format( db.error()) return Response(dumps(response), mimetype=mimetype) if is_url_in_db(db, url): return Response('{"What happened?": "Someone '\ 'tried to submit a duplicate URL."}', mimetype=mimetype) title = url summary = "~?~" try: thing = urlopen(url, timeout=10) soup = BeautifulSoup(thing) title = soup.title.string # Do some dumb summarizing if we can def concat(a, v): return a + " " + v.strip() visible_stuff = filter(visible, soup.findAll(text=True)) summary = reduce(concat, visible_stuff, "")[:900] + "..." except: pass #return Response('{"What happened?": '\ # 'I dunno bs4 messed up somehow."}', # mimetype=mimetype) created_at = int(mktime(datetime.now().utctimetuple())) is_image = url.lower().endswith(("jpg", "jpeg", "gif", "png")) thumbnail = gen_thumbnail_for_url(url, str(created_at)) record = { "created_at": created_at, "title": title, "url": url, "person": person, "summary": summary, "person_color": PERSON_COLORS[random.randint(0, len(PERSON_COLORS) - 1)], "is_image": is_image, "thumbnail": thumbnail, "comment": submitted_title } db.set(created_at, dumps(record)) db.close() return Response('{"What happened?": "MUDADA"}', mimetype=mimetype)
def __enter__( self ): self._cdata = DB(); self._ddata = DB(); try: if self._mode == "r": assert self._cdata.open( self._fn_cdata, DB.OREADER ); elif self._mode == "w": if isfile( self._fn_cdata ): remove( self._fn_cdata ); assert self._cdata.open( self._fn_cdata, DB.OWRITER | DB.OCREATE ); else: assert False; except: if self._cdata is not None: print( str( self._cdata.error() ) ); raise; try: if self._mode == "r": assert self._ddata.open( self._fn_ddata, DB.OREADER ); elif self._mode == "w": if isfile( self._fn_ddata ): remove( self._fn_ddata ); assert self._ddata.open( self._fn_ddata, DB.OWRITER | DB.OCREATE ); else: assert False; except: if self._ddata is not None: print( str( self._ddata.error() ) ); raise; if self._mode == "r": with open( self._fn_meta, 'rb' ) as f: r = pickle_load( f ); self._len_c = r[ "c" ]; self._len_b = r[ "b" ]; self._len_x = r[ "x" ]; self._co = r[ "co" ]; with open( self._fn_icov, 'rb' ) as f: self._icov = pickle_load( f ); return self;
def __init__(self, db_file): #from train import TAG2ID, WORD2ID#, BAYES_RANK #self.ider = WORD2ID self.db = DB() self.db_file = db_file print path.join(KYOTO_DB_PATH,self.db_file) if not self.db.open(path.join(KYOTO_DB_PATH,self.db_file), DB.OWRITER | DB.OCREATE): print >>sys.stderr, "open error: " + str(self.db.error())
class DbKyoto(object): def __init__(self, db_file): self.db = DB() self.db_file = db_file if not self.db.open(db_file, DB.OWRITER | DB.OCREATE): print >> sys.stderr, 'open error: ' + str(self.db.error()) def set(self, txt, po_id): feature_list = feature_md5(txt) for feature in feature_list: key = feature entry = self.get(key) if not entry: val = array('L', [po_id]) if not self.db.set(key, val.tostring()): print >> sys.stderr, 'open error: ' + str(self.db.error()) else: val = array('L') val.fromstring(entry) if po_id not in val: val.append(po_id) self.db.set(key, val.tostring()) return val def get(self, key): po_id = self.db.get(key) result = array('L') if po_id: result.fromstring(po_id) return result
def __init__(self, dir, subdirs=[]): dir += "_zlib" if dir not in dbList: dbList[dir] = DB() dbList[dir].open( dir + ".kch#ops=c#log=" + dir + ".log#logkinds=debu#zcomp=zlib", DB.OWRITER | DB.OCREATE) self.db = dbList[dir] self.key = "/".join(subdirs)
def get_items_last_X_days(db_file, X, munge=True): dates = {} db = DB() if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE): print "Could not open database." X_days_ago = datetime.now() - timedelta(days=X) cur = db.cursor() cur.jump_back() while True: rec = cur.get(False) if not rec: break loaded = loads(rec[1]) unix = float(loaded['created_at']) time = datetime.fromtimestamp(unix) if time > X_days_ago: if munge: date_obj = date(year=time.year, month=time.month, day=time.day) else: date_obj = time # Javascript expects Date.UTC to spit out dates of a certain # length. day_unix = int(mktime(date_obj.timetuple()))*1000 if dates.get(day_unix, False) == False: dates[day_unix] = {loaded["person"]: 1} else: relevant_dict = dates[day_unix] if relevant_dict.get(loaded["person"], False) == False: relevant_dict[loaded["person"]] = 1 else: relevant_dict[loaded["person"]] = relevant_dict[loaded["person"]] + 1 else: break; cur.step_back() cur.disable() db.close() return dates
def get_last_items(db_file, pages=1): items = [] db = DB() if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE): print "Could not open database." cur = db.cursor() cur.jump_back() while len(items) < (pages * FILTER_MAX): rec = cur.get(False) if not rec: break items.append(rec) cur.step_back() cur.disable() db.close() return items
def init_read(self): self.mode = "read" if self.ext == ".csv": self._file = open(self.filename, "rb") elif self.ext == ".json": with open(self.filename, "rb") as f: self._storage = json.load(f) elif self.ext == ".kch": from kyotocabinet import DB self._storage = DB() self._storage.open(self.filename, DB.OREADER) elif self.ext == ".sqlite3": import sqlite3 self._storage = sqlite3.connect(self.filename) self._cursor = self._storage.cursor() else: raise dexy.exceptions.UserFeedback("unsupported extension %s" % self.ext)
def get_all_items(db_file): items = [] db = DB() if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE): print "Could not open database." cur = db.cursor() cur.jump() while True: rec = cur.get(False) if not rec: break items.append(rec) cur.step() cur.disable() db.close() sorted_items_for_viewing = [loads(item[1]) for item in items] return sorted_items_for_viewing
def decorated_function(*args, **kwargs): # Debug if not current_app.config['CACHE']: return f(*args, **kwargs) db = DB() db.open("/tmp/page_cache.kch") res = None fancy = hash("{}{}{}".format(db_meta_info()['count'], request.url, f.func_name)) res = db.get(fancy) if not res: res = f(*args, **kwargs) db.set(fancy, res) db.close() return res
def aggregate_by_hour(db_file): # Initialize the dict with each hour hours = {key: 0 for key in range(0, 24)} db = DB() if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE): print "Could not open database." cur = db.cursor() cur.jump_back() while True: rec = cur.get(False) if not rec: break loaded = loads(rec[1]) unix = float(loaded['created_at']) time = datetime.fromtimestamp(unix) hours[time.hour] = hours[time.hour] + 1 cur.step_back() cur.disable() db.close() hours = [{ 'name': "{}:00".format(key), 'data': [hours[key]] } for key in hours] return hours
def db_meta_info(): meta = {} db = DB() db_file = current_app.config['DB_FILE'] if not db.open("{0}".format(db_file), DB.OREADER): print "Could not open database (meta info)." meta["size"] = db.size() meta["count"] = db.count() db.close() return meta
def gen_thumbnails(db_file): db = DB() if not db.open("{0}".format(db_file), DB.OREADER | DB.OWRITER): sys.exit(1) cur = db.cursor() cur.jump_back() while True: rec = cur.get(False) if not rec: break loaded = loads(rec[1]) is_image = loaded["url"].lower().endswith(("jpg", "jpeg", "gif", "png")) if is_image: print "Thumbnailing {}".format(loaded["url"]) loaded["is_image"] = True try: thumbnail = gen_thumbnail_for_url(loaded["url"], rec[0]) except IOError as e: print "IOError: {}".format(e) print "Save result: {}".format(cur.set_value(dumps(loaded))) cur.step_back() continue if thumbnail: loaded["thumbnail"] = thumbnail print "Thumbnailed {}".format(loaded["url"]) print "Save result: {}".format(cur.set_value(dumps(loaded))) cur.step_back() cur.disable() db.close() return True
def __init__(self, path, truncate=False): """ Open a new connection to a database using the Kyoto Cabinet engine. Args: * path (str): Path to database. Kwargs: * truncate (bool, False): If database should be truncated before opening. """ self.db = DB() self.batchsize = 1000 self.batch = {} dbparams = '.kct#apow=0#bnum=10000000#msiz=' + str(2 << 30) if truncate: result = self.db.open(path + dbparams, DB.OWRITER | DB.OCREATE | DB.OTRUNCATE) else: result = self.db.open(path + dbparams, DB.OWRITER) if not result: raise PathError('DNA outdb open error: %s ' % self.db.error()) exit(1)
def top_things(db_file): urls = {} people = {} graph = {} db = DB() if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE): print "Could not open database. (Top things)" cur = db.cursor() cur.jump_back() while True: rec = cur.get(False) if not rec: break loaded_rec = loads(rec[1]) split = get_domain(loaded_rec) if urls.get(split, False) == False: urls[split] = 1 else: urls[split] = urls[split] + 1 person = loaded_rec['person'] if people.get(person, False) == False: people[person] = 1 else: people[person] = people[person] + 1 if split is not None and split is not "" and \ person is not None and person is not "": # Build a crazy relational graph out of my nosql data if graph.get(split, False) == False: graph[split] = { "is_person": False, "data": [person], "linked_to_count": 1 } elif person not in graph[split]: graph[split]["data"].append(person) graph[split][ "linked_to_count"] = graph[split]["linked_to_count"] + 1 if graph.get(person, False) == False: graph[person] = {"is_person": True, "data": [split]} elif split not in graph[person]: graph[person]["data"].append(split) cur.step_back() cur.disable() db.close() def get_one(x): return x[1] return (sorted(urls.items(), key=get_one, reverse=True), sorted(people.items(), key=get_one, reverse=True), graph)
def get_items_on_page(page, db_file): item_iter = 0 items = [] db = DB() if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE): print "Could not open database." cur = db.cursor() cur.jump_back() while len(items) < FILTER_MAX: rec = cur.get(False) if not rec: break if item_iter >= (FILTER_MAX * page): items.append(rec) item_iter = item_iter + 1 cur.step_back() cur.disable() db.close() return items
def insert_item(url, person, db_file, submitted_title=''): mimetype = "application/json" db = DB() if not db.open("{0}".format(db_file), DB.OWRITER | DB.OCREATE): response = {} response['What happened?'] = "Couldn't open the damn database. Error: {0}".format(db.error()) return Response(dumps(response), mimetype=mimetype) if is_url_in_db(db, url): return Response('{"What happened?": "Someone '\ 'tried to submit a duplicate URL."}', mimetype=mimetype) title = url summary = "~?~" try: thing = urlopen(url, timeout=10) soup = BeautifulSoup(thing) title = '' if len(submitted_title) > 0: title = submitted_title else: title = soup.title.string # Do some dumb summarizing if we can func = lambda a,v: a + " " + v.strip() visible_stuff = filter(visible, soup.findAll(text=True)) summary = reduce(func, visible_stuff, "")[:900] + "..." except: pass #return Response('{"What happened?": '\ # 'I dunno bs4 messed up somehow."}', # mimetype=mimetype) created_at = int(mktime(datetime.now().utctimetuple())) is_image = url.lower().endswith(("jpg", "jpeg", "gif", "png")) thumbnail = gen_thumbnail_for_url(url, str(created_at)) record = { "created_at": created_at, "title": title, "url": url, "person": person, "summary": summary, "person_color": PERSON_COLORS[random.randint(0, len(PERSON_COLORS)-1)], "is_image": is_image, "thumbnail": thumbnail } db.set(created_at, dumps(record)) db.close() return Response('{"What happened?": "MUDADA"}', mimetype=mimetype)
class KDB: def __init__(self, path, truncate=False): """ Open a new connection to a database using the Kyoto Cabinet engine. Args: * path (str): Path to database. Kwargs: * truncate (bool, False): If database should be truncated before opening. """ self.db = DB() self.batchsize = 1000 self.batch = {} dbparams = '.kct#apow=0#bnum=10000000#msiz=' + str(2 << 30) if truncate: result = self.db.open(path + dbparams, DB.OWRITER | DB.OCREATE | DB.OTRUNCATE) else: result = self.db.open(path + dbparams, DB.OWRITER) if not result: raise PathError('DNA outdb open error: %s ' % self.db.error()) exit(1) def get(self, key): """ Retrieve the item with the given `key`. """ return self.db.get(key) def put(self, key, val): """ Put `val` at `key`. Note that disk writing is done in batches, so be sure to call `close` or `flush` to make sure that values are put into the store. """ self.batch[key] = val if len(self.batch) >= self.batchsize: self.flush() def flush(self): """ Write `put` calls to database. """ self.db.set_bulk(self.batch) self.batch = {} def close(self): """ Flush the database and delete the connection to it. """ self.flush() del self.db
def main(): db_file = argv[1] username = argv[2] if not db_file and not username: print "Need db_file and username." return -1 db = DB() if not db.open("{0}".format(db_file), DB.OWRITER): print "Could not open database." return -1 all_keys = [] cur = db.cursor() cur.jump() while True: rec = cur.get(False) if not rec: break loaded = loads(rec[1]) if loaded["person"] == username: all_keys.append(cur.get_key()) cur.step() cur.disable() print "Found {} records.".format(len(all_keys)) for key in all_keys: print "Pending {}...".format(key) if len(argv) > 3 and argv[3] == '--delete': print "Removing {}...".format(key) if not db.remove(key): print "Could not remove key: {}".format(db.error()) db.close()
def dbOpen(name): db = DB() dbpathname = abspath(self.path) + '/' + name + ".kch" if self.create: # if not db.open(abspath(self.path) + '/' + name + ".kch", # DB.OWRITER | DB.OCREATE | DB.OAUTOSYNC | DB.OAUTOTRAN): if not db.open(dbpathname, DB.OWRITER | DB.OCREATE): raise IOError("open error: %s %s" % ( dbpathname, str(db.error()))) # pragma: NO COVER return db else: # if not db.open(abspath(self.path) + '/' + name + ".kch", # DB.OWRITER | DB.OAUTOSYNC | DB.OAUTOTRAN): if not db.open(dbpathname, DB.OWRITER): # pragma: NO COVER raise IOError("open error: %s %s" % ( dbpathname, str(db.error()))) # pragma: NO COVER return db
def get_items_last_X_days(db_file, X, munge=True): dates = {} db = DB() if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE): print "Could not open database." X_days_ago = datetime.now() - timedelta(days=X) cur = db.cursor() cur.jump_back() while True: rec = cur.get(False) if not rec: break loaded = loads(rec[1]) unix = float(loaded['created_at']) time = datetime.fromtimestamp(unix) if time > X_days_ago: if munge: date_obj = date(year=time.year, month=time.month, day=time.day) else: date_obj = time # Javascript expects Date.UTC to spit out dates of a certain # length. day_unix = int(mktime(date_obj.timetuple())) * 1000 if dates.get(day_unix, False) == False: dates[day_unix] = {loaded["person"]: 1} else: relevant_dict = dates[day_unix] if relevant_dict.get(loaded["person"], False) == False: relevant_dict[loaded["person"]] = 1 else: relevant_dict[ loaded["person"]] = relevant_dict[loaded["person"]] + 1 else: break cur.step_back() cur.disable() db.close() return dates
def get_page_count(item_filter=lambda x: True): count = 0 db = DB() db_file = current_app.config['DB_FILE'] if not db.open("{0}".format(db_file), DB.OREADER | DB.OWRITER | DB.OCREATE): print "Could not open database (get_page_count). Error: {}".format( db.error()) cur = db.cursor() cur.jump_back() while True: rec = cur.get(False) if not rec: break if item_filter(rec): count = count + 1 cur.step_back() cur.disable() db.close() return count / FILTER_MAX
def gen_thumbnails(db_file): db = DB() if not db.open("{0}".format(db_file), DB.OREADER | DB.OWRITER): sys.exit(1) cur = db.cursor() cur.jump_back() while True: rec = cur.get(False) if not rec: break loaded = loads(rec[1]) is_image = loaded["url"].lower().endswith( ("jpg", "jpeg", "gif", "png")) if is_image: print "Thumbnailing {}".format(loaded["url"]) loaded["is_image"] = True try: thumbnail = gen_thumbnail_for_url(loaded["url"], rec[0]) except IOError as e: print "IOError: {}".format(e) print "Save result: {}".format(cur.set_value(dumps(loaded))) cur.step_back() continue if thumbnail: loaded["thumbnail"] = thumbnail print "Thumbnailed {}".format(loaded["url"]) print "Save result: {}".format(cur.set_value(dumps(loaded))) cur.step_back() cur.disable() db.close() return True
def get_page_count(item_filter = lambda x: True): count = 0 db = DB() db_file = current_app.config['DB_FILE'] if not db.open("{0}".format(db_file), DB.OREADER | DB.OWRITER | DB.OCREATE): print "Could not open database (get_page_count). Error: {}".format(db.error()) cur = db.cursor() cur.jump_back() while True: rec = cur.get(False) if not rec: break if item_filter(rec): count = count + 1 cur.step_back() cur.disable() db.close() return count / FILTER_MAX
class FeatureSelector( Frontend ): def __init__( self, fn, mode ): Frontend.__init__( self, fn, mode ); self._kdbfn = None; self._kdb = None; self._ldbdn = None; self._ldb = None; self._len_c = None; self._len_b = None; self._len_x = None; self._ic = None; self._icbp = None; self._needs_initialization = True; self._core_dims = set(); self._satellite_dims = set(); self._removed_dims = set(); self._remove_c = set(); self._remove_b = set(); self._remove_x = set(); self.bypass_c = False; self.bypass_b = False; self.bypass_x = False; def __enter__( self ): if self._mode == "r": with open( self._fn, "rb" ) as f: state = pickle_load( f ); self._len_c = state[ "c" ]; self._len_b = state[ "b" ]; self._len_x = state[ "x" ]; self._lenrow = self._len_c + self._len_b + self._len_x; self._ic = state[ "ic" ]; self._icbp = state[ "icbp" ]; if self._mode == "w": with NamedTemporaryFile() as tmpfn: self._kdbfn = tmpfn.name + '.kch'; self._kdb = KDB(); try: assert self._kdb.open( self._kdbfn, KDB.OWRITER | KDB.OCREATE ); except: print( str( self._kdb.error() ) ); raise; with TemporaryDirectory() as tmpdirname: self._ldbdn = tmpdirname; self._ldb = LDB( self._ldbdn, create_if_missing=True ); return self; def __exit__( self, exc_type, exc_value, traceback ): assert Frontend.__exit__( self, exc_type, exc_value, traceback ) == False; if self._ldb is not None: sleep( 3.0 ); self._ldb.close() if self._ldbdn is not None: rmtree( self._ldbdn ); if self._kdb is not None: try: assert self._kdb.close(); except: print( str( self._kdb.error() ) ); raise; if self._kdbfn is not None: remove( self._kdbfn ); def train( self, row ): ( y, c, b, x ) = row; if self._len_c is None: self._len_c = len(c); assert self._len_c == len(c); if self._len_b is None: self._len_b = len(b); assert self._len_b == len(b); if self._len_x is None: self._len_x = len(x); assert self._len_x == len(x); row = c + b + x; if Frontend.train( self, row ): return True; keyfmt = '>IIIII'; for i in range( 0, self._lenrow ): for j in range( 0, self._lenrow ): if ( i >= j ) and ( not ( i == self._lenrow-1 ) ): continue; key = pack( keyfmt, i, j, y, row[i], row[j] ); try: assert self._kdb.increment( key, 1, 0 ); except: print( str(self._kdb.error()) ); raise; def _stats( self, cnt_by_a, cnt_by_b, cnt_by_ab ): h_a = 0.0; h_b = 0.0; h_ab = 0.0; for ( val_a, cnt ) in cnt_by_a.items(): p = float(cnt) / float(self._rowcount); if p > 0.0: h_a -= p * log( p, 2.0 ); for ( val_b, cnt ) in cnt_by_b.items(): p = float(cnt) / float(self._rowcount); if p > 0.0: h_b -= p * log( p, 2.0 ); for( (val_a,val_b), cnt ) in cnt_by_ab.items(): p = float(cnt) / float(self._rowcount); if p > 0.0: h_ab -= p * log( p, 2.0 ); if h_a == 0.0: return 1.0; if h_b == 0.0: return 1.0; mi = h_a + h_b - h_ab; return ( mi / min( h_a, h_b ), h_a, h_b, h_ab, mi ); def _get_info_content_by_dimension( self, i ): keyfmt = '>IIIII'; valfmt = '>Q'; j = None; cnt_by_a = {}; cnt_by_b = {}; cnt_by_ab = {}; total = 0; with self._ldb.iterator() as it: it.seek( pack( keyfmt, i,0,0,0,0 ) ); for ( key, val ) in it: key = unpack( keyfmt, key ); val = unpack( valfmt, val )[ 0 ]; if not ( key[0] == i ): break; if j is None: j = key[1]; if not ( key[1] == j ): break; # key[2] is the y-value a = key[2]; # key[3] is the value for the i-th dimension b = key[3]; cnt_by_ab[ (a,b) ] = cnt_by_ab.get( (a,b), 0 ) + val; cnt_by_a[ a ] = cnt_by_a.get( a, 0 ) + val; cnt_by_b[ b ] = cnt_by_b.get( b, 0 ) + val; total += val; try: assert total == self._rowcount; except: print( i, j, total, self._rowcount ); raise; return self._stats( cnt_by_a, cnt_by_b, cnt_by_ab ); def _get_info_content_by_pair( self, i, j ): keyfmt = '>IIIII'; valfmt = '>Q'; cnt_by_a = {}; cnt_by_b = {}; cnt_by_ab = {}; total = 0; with self._ldb.iterator() as it: it.seek( pack( keyfmt, i,j,0,0,0 ) ); for ( key, val ) in it: key = unpack( keyfmt, key ); val = unpack( valfmt, val )[ 0 ]; if not ( ( key[0] == i ) and ( key[1] == j ) ): break; # key[2] is the y-value, key[3] the i-th value for the i-th dim a = ( key[2], key[3] ); # key[2] is the y-value, key[4] the i-th value for the j-th dim b = ( key[2], key[4] ); assert (a,b) not in cnt_by_ab; cnt_by_ab[ (a,b) ] = cnt_by_ab.get( (a,b), 0 ) + val; cnt_by_a[ a ] = cnt_by_a.get( a, 0 ) + val; cnt_by_b[ b ] = cnt_by_b.get( b, 0 ) + val; total += val; assert total == self._rowcount; return self._stats( cnt_by_a, cnt_by_b, cnt_by_ab ); def _finalize( self ): assert Frontend._finalize( self ) is None; if False: print( "unique combinations = ", self._kdb.count() ); keyfmt = '>IIIII'; valfmt = '>Q'; c = self._kdb.cursor(); c.jump(); gt2 = 0; gt4 = 0; gt8 = 0; gt16 = 0; gt32 = 0; while True: r = c.get( True ); if not r: break; self._ldb.put( r[0], r[1] ); key = unpack( keyfmt, r[0] ); val = unpack( valfmt, r[1] )[ 0 ]; if val > 2: gt2 += 1; if val > 4: gt4 += 1; if val > 8: gt8 += 1; if val > 16: gt16 += 1; if val > 32: gt32 += 1; if False: print( gt2, gt4, gt8, gt16, gt32 ); self._ic = {}; for i in range( 0, self._lenrow ): self._ic[ i ] = self._get_info_content_by_dimension( i ); self._icbp = {}; for i in range( 0, self._lenrow ): for j in range( 0, self._lenrow ): if i >= j: continue; self._icbp[ (i,j) ] = self._get_info_content_by_pair( i, j ); self._state \ = { "ic": self._ic, "icbp": self._icbp, "c": self._len_c, "b": self._len_b, "x": self._len_x }; def _fmt_dim( self, d_ ): d = None; if d_ < self._len_c: d = "c" + str( d_ ); elif d_ < self._len_c + self._len_b: d = "b" + str( d_ - self._len_c ); elif d_ < self._len_c + self._len_b + self._len_x: d = "x" + str( d_ - self._len_c - self._len_b ); else: assert False; return "{:d}({:s})".format( d_, d ); def _init( self ): self._needs_initialization = False; if False: for i in sorted( self._ic ): (corr,h_a,h_b,h_ab,mi) = self._ic[ i ]; print( "{:s} {:1.4f} {:1.4f} {:1.4f} {:1.4f} {:1.4f}"\ .format( self._fmt_dim( i ), corr, h_a, h_b, h_ab, mi ) ); for (i,j) in sorted( self._icbp ): (corr,h_a,h_b,h_ab,mi) = self._icbp[ (i,j) ]; print( "{:s} {:s} {:1.4f} {:1.4f} {:1.4f} {:1.4f} {:1.4f}"\ .format( self._fmt_dim( i ), self._fmt_dim( j ), corr, h_a, h_b, h_ab, mi ) ); entropy \ = [ ( h_ab, i ) \ for ( i, (corr,h_a,h_b,h_ab,mi) ) in self._ic.items() ]; output_correlation \ = [ ( corr, i ) \ for ( i, (corr,h_a,h_b,h_ab,mi) ) in self._ic.items() ]; self._core_dims = set(); self._core_dims \ |= { i \ for ( h_ab, i ) \ in sorted( entropy, reverse=True )[ :5 ] }; self._core_dims \ |= { i \ for ( h_ab, i ) \ in sorted( output_correlation, reverse=True )[ :3 ] }; if True: print( "core = ", " ".join([ self._fmt_dim(d) for d in self._core_dims ]) ); self._satellite_dims = set(); for core_dim in self._core_dims: satellite_dim = None; satellite_dim_c = None; satellite_dim_stats = None; for ( (i,j), (corr,h_a,h_b,h_ab,mi) ) in self._icbp.items(): if corr <= 0.5: continue; other_dim = None; if i == core_dim: other_dim = j; elif j == core_dim: other_dim = i; else: continue; if ( satellite_dim_c is None ) or ( corr > satellite_dim_c ): satellite_dim = other_dim; satellite_dim_c = corr; satellite_dim_stats = (corr,h_a,h_b,h_ab,mi); if satellite_dim is not None: self._satellite_dims.add( satellite_dim ); if False: print( '->', self._fmt_dim(core_dim), self._fmt_dim(satellite_dim) ); print( "{:1.4f} {:1.4f} {:1.4f} {:1.4f} {:1.4f}"\ .format( *(corr,h_a,h_b,h_ab,mi) ) ); if True: print( "satellite = ", " ".join([ self._fmt_dim(d) for d in self._satellite_dims ]) ); self._removed_dims = set(); for i in self._ic: if i not in self._core_dims and i not in self._satellite_dims: self._removed_dims.add( i ); if True: print( "removed = ", " ".join([ self._fmt_dim(d) for d in self._removed_dims ]) ); for d_ in self._removed_dims: if d_ < self._len_c: self._remove_c.add( d_ ); elif d_ < self._len_c + self._len_b: self._remove_b.add( d_ - self._len_c ); elif d_ < self._len_c + self._len_b + self._len_x: self._remove_x.add( d_ - self._len_c - self._len_b ); else: assert False; def apply_c( self, c ): if self.bypass_c: return c; if self._needs_initialization: self._init(); c_ = []; for ( i, cval ) in enumerate( c ): if not i in self._remove_c: c_.append( cval ); return c_; def apply_b( self, b ): if self.bypass_b: return b; if self._needs_initialization: self._init(); b_ = []; for ( i, bval ) in enumerate( b ): if not i in self._remove_b: b_.append( bval ); return b_; def apply_x( self, x ): if self.bypass_x: return x; if self._needs_initialization: self._init(); x_ = []; for ( i, xval ) in enumerate( x ): if not i in self._remove_x: x_.append( xval ); return x_; def __call__( self, row ): if self._needs_initialization: self._init(); ( y, c, b, x ) = row; y_ = y; return \ ( y_, self.apply_c( c ), self.apply_b( b ), self.apply_x( x ) );
def get_user_stats(username, db_file): item = { "username": username, "aliases": [], "total_posts": 0, "domains": {}, "first_post_date": None, "first_post_date_unix": None, "most_recent_post": None, "most_recent_post_unix": 0, "average_posts_per_hour": 0.0, "average_posts_per_day": 0.0, "average_posts_per_week": 0.0 } db = DB() if not db.open("{0}".format(db_file), DB.OREADER | DB.OCREATE): print "Could not open database." cur = db.cursor() cur.jump() while True: rec = cur.get(False) if not rec: break loaded_rec = loads(rec[1]) if loaded_rec['person'] != username: cur.step() continue # Looks like this is a post by the user we're looking for split = get_domain(loaded_rec) if item['domains'].get(split, False) == False: item['domains'][split] = 1 else: item['domains'][split] = item['domains'][split] + 1 if item['first_post_date_unix'] is None: item['first_post_date_unix'] = loaded_rec['created_at'] if item['most_recent_post_unix'] < loaded_rec['created_at']: item['most_recent_post_unix'] = loaded_rec['created_at'] item['total_posts'] = item['total_posts'] + 1 cur.step() cur.disable() db.close() # Clean up everything first_time = None if item['first_post_date_unix'] is not None: unix = float(item['first_post_date_unix']) first_time = datetime.fromtimestamp(unix) item['first_post_date'] = first_time.isoformat() recent_time = None if item['most_recent_post_unix'] is not None: unix = float(item['most_recent_post_unix']) recent_time = datetime.fromtimestamp(unix) item['most_recent_post'] = recent_time.isoformat() if first_time and recent_time: delta = recent_time - first_time item['user_age_days'] = delta.days item['user_age_seconds'] = delta.total_seconds() item['average_posts_per_hour'] = item['total_posts'] / (delta.total_seconds() / 60.0) item['average_posts_per_day'] = item['total_posts'] / (delta.total_seconds() / 60.0 / 24.0) item['average_posts_per_week'] = item['total_posts'] / (delta.total_seconds() / 60.0 / 24.0 / 7.0) return item