def write_aceview_genes(species,log=0): """ **Write correspondance between AceView and Ensembl genes.** :: Arguments: species --- species name chipName -- chip name log ------- handle of a log file for recording messages Input: %species_genes_by_region.bkdb %species_ensembl_genes_by_gene.bkdb Output: %species_ens_by_ace_gene.txt """ #open ace genes by- ensembl region t1=time.time() path=os.environ['PYDATA']+'/'+species+'/ensembl/'+species+'_genes_by_region.bkdb' byRegionDB=bsddb.btopen(path) path=os.environ['PYDATA']+'/'+species+'/aceview/'+species+'_ensembl_genes_by_gene.bkdb' byAceGeneDB=bsddb.btopen(path) outFile=open(os.environ['PYDATA']+"/"+species+'/txt/'+species+'_ens_by_ace_gene.txt','w') byAceGeneIDs=byAceGeneDB.keys() ensGeneIDs=[] for region in byRegionDB: ensGeneIDs.extend(cPickle.loads(byRegionDB[region]).IDs) for byAceGeneID in byAceGeneIDs: ensGenes=cPickle.loads(byAceGeneDB[byAceGeneID]) outFile.write(byAceGeneID+'\t') if ensGenes==[] or ensGenes==None: outFile.write('{''}\n') else: outFile.write('{') for i in range(len(ensGenes)): try: #current ensemnl gene may have been already erased ensGeneIDs.pop(ensGeneIDs.index(ensGenes[i])) except: pass if i==0: outFile.write("'"+ensGenes[i]+"'") else: outFile.write(','+"'"+ensGenes[i]+"'") outFile.write('}\n') for i in range(len(ensGeneIDs)): outFile.write('0\t%s\n'%ensGeneIDs[i]) byRegionDB.close() byAceGeneDB.close() outFile.close() t2=time.time() if log!=0: log.write('%s\t%s\t\t30\t%s_aceview_genes.txt\taceview\t%u\t%.2f\n'%(date.today(),species,species,len(byAceGeneIDs),t2-t1))
def _initcache(self, cachefile_name=None): if cachefile_name: self._id = cachefile_name else: self._id = "".join( [uuid.uuid4().hex, time.strftime("%Y%m%d%H%M%S")]) self._cache_path = path.dirname(__file__) if len(self._cache_path) == 0: self._cache_path = os.getcwd() assert path.exists(self._cache_path) if cachefile_name: if "/".join([self._cache_path, self._id]) not in self.__class__.__setitem__mutex__: self.data = bsddb.btopen( "/".join([self._cache_path, self._id]), "c") self.__class__.__setitem__mutex__.setdefault( "/".join([self._cache_path, self._id]), [threading.Lock(), self.data, 0]) self.__class__.__setitem__mutex__["/".join( [self._cache_path, self._id])][2] += 1 else: self.data = self.__class__.__setitem__mutex__["/".join( [self._cache_path, self._id])][1] self.__class__.__setitem__mutex__["/".join( [self._cache_path, self._id])][2] += 1 else: self.data = bsddb.btopen("/".join([self._cache_path, self._id]), "n") self.__class__.__setitem__mutex__.setdefault( "/".join([self._cache_path, self._id]), [threading.Lock(), self.data, 0])
def __init__(self, filename, qsize, scd_policy, lbd=0.7, upd_cl_us=None): self.filename = filename self.scd_policy = scd_policy index_vertices_file = os.path.splitext(filename)[0] + "_v.hdr" index_isolated_vertices_file = os.path.splitext(filename)[0] + "_iv.hdr" index_corner_file = os.path.splitext(filename)[0] + "_c.hdr" index_corner_vertice_file = os.path.splitext(filename)[0] + "_cv.hdr" index_clusters_file = os.path.splitext(filename)[0] + "_cl.hdr" self.index_vertices = bsddb.btopen(index_vertices_file) self.index_isolated_vertices = bsddb.btopen(index_isolated_vertices_file) self.index_corners = bsddb.btopen(index_corner_file) self.index_corner_vertice = bsddb.btopen(index_corner_vertice_file) self.index_clusters = bsddb.btopen(index_clusters_file) self.iv_keys = sorted([int(i) + 1 for i in self.index_vertices.keys()]) self.ic_keys = sorted([int(i) + 1 for i in self.index_corners.keys()]) # self.icv_keys = sorted([int(i) + 1 for i in self.index_corner_vertice.keys()]) self.cfile = open(filename) self.load_header() self.cl_usage = {} self.queue_size = int(math.ceil((qsize / 100.0 * len(self.index_clusters)))) self.timestamp = 0 self.lbd = lbd self.wastings = [] self.last_removed = -1 self.last_loaded = -1 self._n_load_clusters = {} self._n_unload_clusters = {} self.misses = 0 self.hits = 0 self.access = 0 self.colour = 0 self.__vertices = {} self.__L = {} self.__R = {} self.__O = {} self.__V = {} self.__C = {} self.__VOs = {} self.vertices = _DictGeomElem(self, "vertices", self.__vertices) self.L = _DictGeomElem(self, "L", self.__L) self.R = _DictGeomElem(self, "R", self.__R) self.O = _DictGeomElem(self, "O", self.__O) self.V = _DictGeomElem(self, "V", self.__V) self.C = _DictGeomElem(self, "C", self.__C) self.VOs = _DictGeomElem(self, "VOs", self.__VOs) if upd_cl_us is None: self.update_cluster_usage = self._update_cluster_usage else: self.update_cluster_usage = types.MethodType(upd_cl_us, self) signal.signal(signal.SIGINT, lambda x, y: self.print_cluster_info())
def openbase(self): if sys.platform.startswith('darwin'): self.dbcmdscheduler = plyvel.DB(self.name_basesession, create_if_missing=True) self.dbsessionscheduler = plyvel.DB(self.name_basecmd, create_if_missing=True) else: self.dbcmdscheduler = bsddb.btopen(self.name_basecmd , 'c') self.dbsessionscheduler = bsddb.btopen(self.name_basesession , 'c')
def main(): try: db = bsddb.btopen(DA_FILE, "w") except: print "DB doesn't exist, creating a new one" db = bsddb.btopen(DA_FILE, "c") lib.set_seed(SEED) for index in range(DB_SIZE): krng = 64 + lib.get_random() % 64 key = "" for i in range(krng): key += str(unichr(lib.get_random_char())) vrng = 64 + lib.get_random() % 64 value = "" for i in range(vrng): value += str(unichr(lib.get_random_char())) print key print value print "" db[key] = value try: db.close() except Exception as e: print e
def OpenDB(): global db db = bsddb.btopen(DBFILE, 'c') try: for k in db.keys(): None except: db.close() db = bsddb.btopen(DBFILE, 'n')
def __init__(self, resource_prefix): """ Init the knowledge resource :param resource_prefix - the resource directory and file prefix """ self.term_to_id = bsddb.btopen(resource_prefix + '_term_to_id.db', 'r') self.id_to_term = bsddb.btopen(resource_prefix + '_id_to_term.db', 'r') self.path_to_id = bsddb.btopen(resource_prefix + '_path_to_id.db', 'r') self.id_to_path = bsddb.btopen(resource_prefix + '_id_to_path.db', 'r') self.l2r_edges = bsddb.btopen(resource_prefix + '_l2r.db', 'r')
def __init__(self, resource_prefix): """ Init the knowledge resource :param resource_prefix - the resource directory and file prefix """ self.term_to_id = bsddb.btopen(resource_prefix + '_term_to_id.db') self.id_to_term = bsddb.btopen(resource_prefix + '_id_to_term.db') self.path_to_id = bsddb.btopen(resource_prefix + '_path_to_id.db') self.id_to_path = bsddb.btopen(resource_prefix + '_id_to_path.db') self.l2r_edges = bsddb.btopen(resource_prefix + '_l2r.db')
def _initcache(self, cachefile_name = None): if cachefile_name: self._id = cachefile_name else: self._id = "".join([uuid.uuid4().hex, time.strftime("%Y%m%d%H%M%S")]) self._cache_path = path.dirname(__file__) assert path.exists(self._cache_path) if cachefile_name: self.data = bsddb.btopen(os.path.join(self._cache_path, self._id), "w") else: self.data = bsddb.btopen(os.path.join(self._cache_path, self._id), "n")
def __init__(self, filename, mode): self.mode = mode if mode == "w": self.test_db_items = bsddb.btopen(filename, 'n') elif mode == "r": self.test_db_items = bsddb.btopen(filename, mode) self.next_rec_num = 0 # Initialise next record counter self.num_records = len(self.test_db_items)
def __openDB4(self): """Make sure the database is read, and open all subdatabases. Raise bsddb.error.""" dbpath = self._getDBPath() if not os.path.isdir(dbpath): os.makedirs(dbpath) if self.dbopen: return self.OK # We first need to remove the __db files, otherwise rpm will later # be really upset. :) for i in xrange(9): try: os.unlink(os.path.join(dbpath, "__db.00%d" % i)) except OSError: pass try: self.basenames_db = bsddb.hashopen( os.path.join(dbpath, "Basenames"), "c") self.conflictname_db = bsddb.hashopen( os.path.join(dbpath, "Conflictname"), "c") self.dirnames_db = bsddb.btopen( os.path.join(dbpath, "Dirnames"), "c") self.filemd5s_db = bsddb.hashopen( os.path.join(dbpath, "Filemd5s"), "c") self.group_db = bsddb.hashopen( os.path.join(dbpath, "Group"), "c") self.installtid_db = bsddb.btopen( os.path.join(dbpath, "Installtid"), "c") self.name_db = bsddb.hashopen( os.path.join(dbpath, "Name"), "c") self.packages_db = bsddb.hashopen( os.path.join(dbpath, "Packages"), "c") self.providename_db = bsddb.hashopen( os.path.join(dbpath, "Providename"), "c") self.provideversion_db = bsddb.btopen( os.path.join(dbpath, "Provideversion"), "c") self.requirename_db = bsddb.hashopen( os.path.join(dbpath, "Requirename"), "c") self.requireversion_db = bsddb.btopen( os.path.join(dbpath, "Requireversion"), "c") self.sha1header_db = bsddb.hashopen( os.path.join(dbpath, "Sha1header"), "c") self.sigmd5_db = bsddb.hashopen( os.path.join(dbpath, "Sigmd5"), "c") self.triggername_db = bsddb.hashopen( os.path.join(dbpath, "Triggername"), "c") self.dbopen = True except bsddb.error: return return self.OK
def save_clusters(lr, clusters, filename): with file(filename, "w") as cfile: # indexes index_vertices_file = os.path.splitext(filename)[0] + "_v.hdr" index_isolated_vertices_file = os.path.splitext(filename)[0] + "_iv.hdr" index_clusters_file = os.path.splitext(filename)[0] + "_cl.hdr" index_corner_file = os.path.splitext(filename)[0] + "_c.hdr" index_corner_vertice_file = os.path.splitext(filename)[0] + "_cv.hdr" index_vertices = bsddb.btopen(index_vertices_file) index_isolated_vertices = bsddb.btopen(index_isolated_vertices_file) index_corners = bsddb.btopen(index_corner_file) index_corner_vertice = bsddb.btopen(index_corner_vertice_file) index_clusters = bsddb.btopen(index_clusters_file) cfile.write("edge vertex: %d\n" % lr.mr) cfile.write("vertex: %d\n" % lr.m) cfile.write("triangles: %d\n" % lr.number_triangles) for i, cluster in enumerate(clusters): # cfile.write("Cluster %d\n" % i) init_cluster = cfile.tell() minv, maxv = 2 ** 32, -1 minc, maxc = 2 ** 32, -1 mincv, maxcv = 2 ** 32, -1 for elem in cluster: if elem[0] == "v": if elem[1] < lr.mr: maxv = max(elem[1], maxv) minv = min(elem[1], minv) else: index_isolated_vertices[str(elem[1])] = str(i) elif elem[0] == "V": index_corners[str(elem[1])] = str(i) elif elem[0] == "C": # maxcv = max(elem[1], maxcv) # mincv = min(elem[1], mincv) index_corner_vertice[str(elem[1])] = str(i) cfile.write(" ".join([str(e) for e in elem]) + "\n") cluster_size = cfile.tell() - init_cluster if maxv > -1: index_vertices[str(maxv)] = str(i) # if maxc > -1: # index_corners[str(maxc)] = str(i) # if maxcv > -1: # index_corner_vertice[str(maxcv)] = str(i) index_clusters[str(i)] = "%d %d %d %d" % (init_cluster, cluster_size, minv, maxv)
def __init__(self, filename, stl_filename): self.filename = filename self.stl_filename = stl_filename index_vertices_file = os.path.splitext(filename)[0] + '_v.hdr' index_clusters_file = os.path.splitext(filename)[0] + '_c.hdr' self.index_vertices = bsddb.btopen(index_vertices_file) self.index_clusters = bsddb.btopen(index_clusters_file) self.loaded_clusters = [] self.faces = {} self.vertices = {} self._make_clusters_maps()
def __init_dbd__(self): print "prepare to initial the bdb" print "[#] Check if the db exist" if os.path.exists("all_sites.db"): os.remove('all_sites.db') if os.path.exists('visited.db'): os.remove('visited.db') print "[#] Cleared the db" """ >>> import bsddb >>> db = bsddb.btopen('spam.db', 'c') >>> for i in range(10): db['%d'%i] = '%d'% (i*i) ... >>> db['3'] '9' >>> db.keys() ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'] >>> db.first() ('0', '0') >>> db.next() ('1', '1') >>> db.last() ('9', '81') >>> db.set_location('2') ('2', '4') >>> db.previous() ('1', '1') >>> for k, v in db.iteritems(): ... print k, v 0 0 1 1 2 4 3 9 4 16 5 25 6 36 7 49 8 64 9 81 >>> '8' in db True >>> db.sync() 0 """ try: self.all_sites = bsddb.btopen(file = 'all_sites.db', flag = 'c') self.visited = bsddb.btopen(file = 'visited.db', flag = 'c') print "[*]Success init BDB" except: print "[!]Bad ! Can't create BDB!"
def __init_dbd__(self): try: index = "result_server" + str(time.strftime('%Y%m%d%H%M%S')) + ".db" self.scan_report = bsddb.btopen(file = index,flag = 'c') print "result_server is created successfully!" self.filename.append(index) self.filelist = bsddb.btopen(file = "filelist_re.db",flag = 'c') self.filelist[str(time.strftime('%Y%m%d%H%M%S'))] = index # print self.filelist.items() except: print "Can't create BDB files!"
def open(self): file_flag = lambda fname: 'c' if (not os.path.exists(fname)) else 'rw' if (not self.isOpened): if ((self.has_bsddb) and ((self.isPickleMethodUseStrings) or (self.isPickleMethodUseMarshal) or (self.isPickleMethodUseSafeSerializer))): self.__db = bsddb.btopen(self.fileName, file_flag(self.fileName)) self.__isOpened = True elif (self.isPickleMethodUseBsdDbShelf): self.__db = shelve.BsdDbShelf( bsddb.btopen(self.fileName, file_flag(self.fileName)), 0, True) self.__isOpened = True
def generate_dbs(): db_i_table = bsddb.btopen("../components/data/i_table.db", 'c') db_n_table = bsddb.btopen("../components/data/n_table.db", 'c') i_table, n_table = _make_table() db_i_table.update(i_table) db_n_table.update(n_table) db_i_table.sync() db_n_table.sync() db_i_table.close() db_n_table.close()
def __init__(self): self.workDir = db_config.workDir self.osmsis = "/home/dndred/osmosis/bin/osmosis" self.useRealDataCache = True self.useOsmDataCache = True self.useGeoCodeCache = True self.saveAnonymousObj = True self.osmSearch = [] # self.conGis = psycopg2.connect(host=db_config.osm_host, database=db_config.osm_database, user=db_config.osm_user, password=db_config.osm_password) # self.curGis = self.conGis.cursor(cursor_factory=psycopg2.extras.RealDictCursor) self.bdb = bsddb.btopen(db_config.bdbname, 'c') self.APIbdb = bsddb.btopen(db_config.ApiCache, 'c') self.osmObjects = [] self.realObjects = []
def generate_dbs(trunc_limit): cur_path = os.path.dirname(os.path.abspath(__file__)) + "/../" + conf.DB_DIR + "/" db_i_table = bsddb.btopen(cur_path + conf.FN_I_TABLE[:-3] + "_" + str(trunc_limit) + ".db", 'c') db_n_table = bsddb.btopen(cur_path + conf.FN_N_TABLE[:-3] + "_" + str(trunc_limit) + ".db", 'c') i_table, n_table = _make_table(trunc_limit) db_i_table.update(i_table) db_n_table.update(n_table) db_i_table.sync() db_n_table.sync() db_i_table.close() db_n_table.close()
def __init__bsddb__(self): print "Create the bsddb!!" print "[^] checking if the bsddb is exist" if os.path.exists("pages.db"): print "[*] Delete the exist db!" os.remove("pages.db") if os.path.exists("visited.db"): print "[*] Delete the visited.db" os.remove("visited.db") try: self.pages = bsddb.btopen(file="pages.db", flag='c') self.visited = bsddb.btopen(file="visited.db", flag='c') print "[^] create db success!!" except: print "Create db error!!"
def __init__bsddb__(self): print "Create the bsddb!!" print "[^] checking if the bsddb is exist" if os.path.exists("pages.db"): print "[*] Delete the exist db!" os.remove("pages.db") if os.path.exists("visited.db"): print "[*] Delete the visited.db" os.remove("visited.db") try: self.pages=bsddb.btopen(file="pages.db",flag='c') self.visited=bsddb.btopen(file="visited.db",flag='c') print "[^] create db success!!" except: print "Create db error!!"
def generate_dbs(): cur_path = os.path.dirname(os.path.abspath(__file__)) + "/../" + conf.DB_DIR + "/" db_i_table = bsddb.btopen(cur_path + conf.FN_I_TABLE, 'c') db_n_table = bsddb.btopen(cur_path + conf.FN_N_TABLE, 'c') i_table, n_table = _make_table() db_i_table.update(i_table) db_n_table.update(n_table) db_i_table.sync() db_n_table.sync() db_i_table.close() db_n_table.close()
def CheckBackup(self): """ Checks the backup file. """ try: # Let's see if the backup is fine... db = bsddb.btopen(self.dbName + ".bak", "c") keys = db.keys() db.close() os.remove(self.dbName) shutil.copy(self.dbName + ".bak", self.dbName) self.db = bsddb.btopen(self.dbName, "c") return True except: # The backup database is corrupted... bad news return False
def test_bsddb(): import bsddb db = bsddb.btopen("bsddb.db", "c") db["key1"] = "value1" db["key2"] = "value2" db.close() db = bsddb.btopen("bsddb.db", "r") if len(db) != 2: raise Exception("Wrong length") if db["key1"] != "value1": raise Exception("value1 incorrect {}".format(db["key1"])) if db["key2"] != "value2": raise Exception("value2 incorrect {}".format(db["key2"]))
def execute(self, op): if op == 1: autor_db = bsddb.btopen('autor.db', 'c') if self._cod not in autor_db: print("Codigo nao existente, por favor, tente novamente.\n") return autor_db.pop(self._cod) autor_db.close() if op == 2: leitor_db = bsddb.btopen('leitor.db', 'c') if self._cod not in leitor_db: print("Codigo nao existente, por favor, tente novamente.\n") return leitor_db.pop(self._cod) leitor_db.close() if op == 3: livros_db = bsddb.btopen('livros.db', 'c') print(self._cod) if self._cod not in livros_db: print("Codigo nao existente, por favor, tente novamente.\n") return livros_db.pop(self._cod) livros_db.close() if op == 4: autor_livro_db = bsddb.btopen('autor_livro.db', 'c') if (self._codL in autor_livro_db): autor_livro_db.pop(self._codL) else: print("Codigo do livro invalido.\n") return autor_livro_db.close()
def create(user, path, name): """ insere un fichier 'name' dans le FilesPool se trouvant dans 'path' ce fichier appartien a l'utilisateur 'user' """ db = bsddb.btopen(folder + 'index.db', 'c') if not os.path.exists(path): raise ("Le fichier n'existe pas") basename = name basename = basename.replace('_', '') #on supprime les '_' du user user = user.replace('_', '') #generation de la cle de la base berkeley key = user + '_' + str(random.getrandbits(30)) + '_' + basename while db.get(key): key = user + '_' + str(random.getrandbits(30)) + '_' + basename #generation du chemin dans l'arborescence hash = md5.new(key).hexdigest() hash = hash[:4] + '/' + hash[4:8] + '/' + hash[8:] while os.path.exists(folder + hash): hash = md5.new(key + random.getrandbits(30)).hexdigest() hash = hash[:4] + '/' + hash[4:8] + '/' + hash[8:] #creation du fichier os.makedirs(folder + hash[:10]) # print 'path=', path, 'folder+hash=', folder+hash #attention copy de shutil pose probleme ! os.rename(path, folder + hash) #sauvegarde du chemin dans la base berkeley db[key] = hash db.close()
def __access_buckets(filename,clear,new_key=None,new_value=None): """ Access data in forkbomb cache, potentially clearing or modifying it as required. """ internal_db = bsddb.btopen(filename, 'c', 0644 ) handle = open(filename,"r") fcntl.flock(handle.fileno(), fcntl.LOCK_EX) storage = shelve.BsdDbShelf(internal_db) if clear: storage.clear() storage.close() fcntl.flock(handle.fileno(), fcntl.LOCK_UN) return {} if not storage.has_key("data"): storage["data"] = {} else: pass if new_key is not None: # bsdb is a bit weird about this newish = storage["data"].copy() newish[new_key] = new_value storage["data"] = newish rc = storage["data"].copy() storage.close() fcntl.flock(handle.fileno(), fcntl.LOCK_UN) return rc
def kbabel_tm_import(self): """Attempt to import the Translation Memory used in KBabel.""" if bsddb is None or not path.exists(self.kbabel_dir): return for tm_filename in os.listdir(self.kbabel_dir): if not tm_filename.startswith( "translations.") or not tm_filename.endswith(".db"): continue tm_file = path.join(self.kbabel_dir, tm_filename) lang = tm_filename.replace("translations.", "").replace(".db", "") translations = bsddb.btopen(tm_file, 'r') for source, target in translations.iteritems(): unit = {"context": ""} source = source[:-1] # null-terminated target = target[16:-1] # 16 bytes of padding, null-terminated unit["source"] = _prepare_db_string(source) unit["target"] = _prepare_db_string(target) self.tmdb.add_dict(unit, "en", lang, commit=False) self.tmdb.connection.commit() logging.debug('%d units migrated from KBabel %s TM.' % (len(translations), lang)) translations.close() self.migrated.append(_("KBabel's Translation Memory: %(database_language_code)s") % \ {"database_language_code": lang})
def open_bsddb(filename, flag='r', useHash=False, mode=0666): """open bsddb index instead of hash by default. useHash=True forces it to use anydbm default (i.e. hash) instead. Also gives more meaningful error messages.""" try: # 1ST OPEN AS BTREE if useHash: # FORCE IT TO USE HASH INSTEAD OF BTREE return open_anydbm(filename, flag) else: return bsddb.btopen(filename, flag, mode) except bsddb.db.DBAccessError: # HMM, BLOCKED BY PERMISSIONS if flag=='c' or flag=='w': # TRY OPENING READ-ONLY try: ifile = file(filename) except IOError: # Hmm, not even readable. Raise a generic permission error. raise PermissionsError('insufficient permissions \ to open file: ' + filename) ifile.close() # We can read the file, so raise a ReadOnlyError. raise ReadOnlyError('file is read-only: '+ filename) else: # r OR n FLAG: JUST RAISE EXCEPTION raise PermissionsError('insufficient permissions to open file: ' + filename) except bsddb.db.DBNoSuchFileError: raise NoSuchFileError('no file named: ' + filename) except bsddb.db.DBInvalidArgError: # NOT A BTREE FILE... try: if useHash: # NO POINT IN TRYING HASH YET AGAIN... raise bsddb.db.DBInvalidArgError # fallback to using default: hash file return open_anydbm(filename, flag) except bsddb.db.DBInvalidArgError: raise WrongFormatError('file does not match expected \ shelve format: ' + filename)
def getTablesWithForm(self): tableList = [] reports = db.btopen(self._fileName) tableList = reports.keys() reports.close() return tableList
def readDb(inFile): if not os.path.exists(inFile): print "error:db file:%s not found" % inFile return False, "" db = bsddb.btopen(inFile, "r") #只读(read only) return True, db
def __init__(self, path="bsd.db"): self.path = path try: os.makedirs(self.path) except: pass self.db = bsddb.btopen(os.path.join(self.path, "datastore.db"), "c")
def make_bsddb(dbfile, dump_file): import bsddb db = bsddb.btopen(dbfile, 'w', cachesize=1024*1024*1024) from infogami.infobase.utils import flatten_dict indexable_keys = set([ "authors.key", "works.key", # edition "authors.author.key", "subjects", "subject_places", "subject_people", "subject_times" # work ]) for type, key, revision, timestamp, json in read_tsv(dump_file): db[key] = json d = simplejson.loads(json) index = [(k, v) for k, v in flatten_dict(d) if k in indexable_keys] for k, v in index: k = web.rstrips(k, ".key") if k.startswith("subject"): v = '/' + v.lower().replace(" ", "_") dbkey = web.safestr('by_%s%s' % (k, v)) if dbkey in db: db[dbkey] = db[dbkey] + " " + key else: db[dbkey] = key db.close() log("done")
def __init__(self, name, shelved=False, queue=False): global _config; self.name = name self.shelved = shelved self.queue = queue self.condition = Condition() self.keys = {} # key self.indexs = {} # 对tuple【2】的索引 self.count = 0 self.scheme=None if shelved: # open db path = '%s/%s_%s.bsd' % (_config['root_dir'], _config['server_key'], name) if self.queue: bsd = bsddb.rnopen(path, 'c') else: bsd = bsddb.btopen(path, 'c') self.db = shelve.BsdDbShelf(bsd) # self.db['中文']='test' # print self.db.keys() # del self.db['中文'] else: if self.queue: self.db = OrderedDict() # memery db else: self.db = {} # memery db
def process_redirect_dump(writer, redirects_dump): import bsddb db = bsddb.btopen('solrdump/redirects.db', 'w', cachesize=1024 * 1024 * 1024) for type, key, revision, timestamp, json_data in read_tsv(redirects_dump): d = json.loads(json_data) if not key.startswith(("/authors/", "/works/")): continue location = d.get('location') if location: # Old redirects still start with /a/ instead of /authors/. location = location.replace("/a/", "/authors/") db[key] = location for key in db: if key.startswith("/works/"): redirect = find_redirect(db, key) if redirect: writer.write([(redirect, "redirect", key)]) return db
def create(user, path, name): """ insere un fichier 'name' dans le FilesPool se trouvant dans 'path' ce fichier appartien a l'utilisateur 'user' """ db = bsddb.btopen(folder+'index.db', 'c') if not os.path.exists(path): raise("Le fichier n'existe pas") basename = name basename = basename.replace('_', '') #on supprime les '_' du user user = user.replace('_', '') #generation de la cle de la base berkeley key = user+'_'+str(random.getrandbits(30))+'_'+basename while db.get(key): key = user+'_'+str(random.getrandbits(30))+'_'+basename #generation du chemin dans l'arborescence hash = md5.new(key).hexdigest() hash = hash[:4]+'/'+hash[4:8]+'/'+hash[8:] while os.path.exists(folder+hash): hash = md5.new(key+random.getrandbits(30)).hexdigest() hash = hash[:4]+'/'+hash[4:8]+'/'+hash[8:] #creation du fichier os.makedirs(folder+hash[:10]) # print 'path=', path, 'folder+hash=', folder+hash #attention copy de shutil pose probleme ! os.rename(path, folder+hash) #sauvegarde du chemin dans la base berkeley db[key] = hash db.close()
def __init__(self, name, path="", unique=True): # # Constructs new DiskQueueManager # # @param name : name of the queue # @param path : path where databases will be stored # @param unique : if set to False, disable unique capability, # also add() begin to work exactly as set() # # @param ext_tree : file extention of tree db # @param ext_hash : file extention of hash db ext_tree = ".tree.db" ext_hash = ".hash.db" self.name = name file_md5 = self._hash(name) self.file_tree = path + file_md5 + ext_tree self.file_hash = path + file_md5 + ext_hash self.db_tree = bsddb.btopen(self.file_tree, "c") self.db_hash = bsddb.hashopen(self.file_hash, "c") self.new_key_id = 0L self.unique = unique
def __init__(self, filename, call_class, key=lambda x: x.id, allow_multiple=False): self.filename = filename self.allow_multiple = allow_multiple self.fh = open(self.filename) self.call_class = call_class self.key = key self.idxfile = filename + FileIndex.ext if need_update(filename, self.idxfile): self.clear() self.db = bsddb.btopen(self.idxfile, 'c') self.create() self.db.close() self.db = bsddb.btopen(self.idxfile, 'r')
def _handleUrl(self, irc, channel, nick, url): # 1 lookup from the db scname = '%s_%s' % (irc.network, channel) if not UrlReader.databases.has_key(scname): # new db dbpath = plugins.makeChannelFilename('%s.db' % scname, 'urldata') UrlReader.databases[scname] = bsddb.btopen(dbpath, 'c') urldb = UrlReader.databases[scname] if urldb.has_key(url): poster, title = pickle.loads(urldb[url]) msg = '%s has already posted it: %s' % (poster, title) irc.reply(msg.encode('utf-8')) else: try: title = self._getTitle(url) if title != None: urldb[url] = pickle.dumps([nick.decode('utf-8'), title]) urldb.sync() irc.reply(title.encode('utf-8')) except: traceback.print_exc() irc.reply('No Title')
def make_bsddb(dbfile, dump_file): import bsddb db = bsddb.btopen(dbfile, "w", cachesize=1024 * 1024 * 1024) indexable_keys = { "authors.key", "works.key", # edition "authors.author.key", "subjects", "subject_places", "subject_people", "subject_times", # work } for type, key, revision, timestamp, json_data in read_tsv(dump_file): db[key] = json_data d = json.loads(json_data) index = [(k, v) for k, v in flatten_dict(d) if k in indexable_keys] for k, v in index: k = web.rstrips(k, ".key") if k.startswith("subject"): v = "/" + v.lower().replace(" ", "_") dbkey = web.safestr(f"by_{k}{v}") if dbkey in db: db[dbkey] = db[dbkey] + " " + key else: db[dbkey] = key db.close() log("done")
def _LoadBdbCacheFile(self, data): """Load data from bdb caches into a map. Args: data: a map.Map subclass Returns: Nothing. Cache data is loaded into the 'data' parameter. Raises: CacheNotFound: if the database file does not exist """ db_file = os.path.join(self.output_dir, self.CACHE_FILENAME) if not os.path.exists(db_file): self.log.debug('cache file does not exist: %r', db_file) raise error.CacheNotFound('cache file does not exist: %r' % db_file) db = bsddb.btopen(db_file, 'r') for k in db: if self.IsMapPrimaryKey(k): password_entry = self.ConvertValueToMapEntry(db[k]) if not data.Add(password_entry): self.log.warn('could not add entry built from %r', db[k]) db.close()
def join_bsddb(file1, file1_key_column, file2, file2_key_column, separator): """ Join two files with the given key columns and separator. Read the first file in fully to create a lookup table. Then iterate through the second file and if the key column from file2 is found in the lookup table output a line for each match from file1. The columns output will be the join field, all the columns from file2 followed by all the columns from file1. An iterator is returned. Each line does not have a trailing newline """ lookup_table = bsddb.btopen(None, "c") for line in file1: line_parts = line.rstrip("\n").split(separator) key = line_parts.pop(file1_key_column) lookup_table[key] = separator.join(line_parts) for line in file2: line_parts = line.rstrip("\n").split(separator) key = line_parts.pop(file2_key_column) if key in lookup_table: (_key, value) = lookup_table.set_location(key) while key == _key: yield separator.join([key, separator.join(line_parts), value]) try: (_key, value) = lookup_table.next() except bsddb.db.DBNotFoundError: (_key, value) = (None, None) lookup_table.close()
def make_bsddb(dbfile, dump_file): import bsddb db = bsddb.btopen(dbfile, 'w', cachesize=1024 * 1024 * 1024) from infogami.infobase.utils import flatten_dict indexable_keys = set([ "authors.key", "works.key", # edition "authors.author.key", "subjects", "subject_places", "subject_people", "subject_times" # work ]) for type, key, revision, timestamp, json in read_tsv(dump_file): db[key] = json d = simplejson.loads(json) index = [(k, v) for k, v in flatten_dict(d) if k in indexable_keys] for k, v in index: k = web.rstrips(k, ".key") if k.startswith("subject"): v = '/' + v.lower().replace(" ", "_") dbkey = web.safestr('by_%s%s' % (k, v)) if dbkey in db: db[dbkey] = db[dbkey] + " " + key else: db[dbkey] = key db.close() log("done")
def dbread(filename): import bsddb db = bsddb.btopen(filename,'r') dump = '' for key in sorted(db.keys()): dump += "%s = %s\n" % (e(key), e(db[key])) return dump
def kbabel_tm_import(self): """Attempt to import the Translation Memory used in KBabel.""" if not path.exists(self.kbabel_dir): return for tm_filename in os.listdir(self.kbabel_dir): if not tm_filename.startswith("translations.") or not tm_filename.endswith(".db"): continue tm_file = path.join(self.kbabel_dir, tm_filename) lang = tm_filename.replace("translations.", "").replace(".db", "") translations = bsddb.btopen(tm_file, "r") for source, target in translations.iteritems(): unit = {"context": ""} source = source[:-1] # null-terminated target = target[16:-1] # 16 bytes of padding, null-terminated unit["source"] = _prepare_db_string(source) unit["target"] = _prepare_db_string(target) self.tmdb.add_dict(unit, "en", lang, commit=False) self.tmdb.connection.commit() logging.debug("%d units migrated from KBabel %s TM." % (len(translations), lang)) translations.close() self.migrated.append( _("KBabel's Translation Memory: %(database_language_code)s") % {"database_language_code": lang} )
def updateDicc(self, tabla_nombre, diccHeader, diccRows): datos = diccHeader datos.append(diccRows) dicc = db.btopen(self._fileName) dicc[tabla_nombre] = pickle.dumps(datos) dicc.close() return 0
def testVerifyFailure(self): # Hide the warning that we expect to get class TestFilter(logging.Filter): def filter(self, record): return not record.msg.startswith('verify failed: %d keys missing') fltr = TestFilter() logging.getLogger('NssDbShadowHandler').addFilter(fltr) # create a map m = shadow.ShadowMap() s = shadow.ShadowMapEntry() s.name = 'foo' self.failUnless(m.Add(s)) updater = nssdb.NssDbShadowHandler({'dir': self.workdir, 'makedb': '/usr/bin/makedb'}) written = updater.Write(m) self.failUnless(os.path.exists(updater.temp_cache_filename), 'updater.Write() did not create a file') # change the cache db = bsddb.btopen(updater.temp_cache_filename) del db[db.first()[0]] db.sync() db.close() retval = updater.Verify(written) self.failUnlessEqual(False, retval) self.failIf(os.path.exists(os.path.join(updater.temp_cache_filename))) # no longer hide this message logging.getLogger('NssDbShadowHandler').removeFilter(fltr)
def readDb(inFile): if not os.path.exists(inFile): print "error:db file:%s not found" % inFile return False , "" db=bsddb.btopen(inFile,"r") #只读(read only) return True, db
def _handleUrl(self,irc,channel,nick,url): # 1 lookup from the db scname='%s_%s'%(irc.network,channel) if not UrlReader.databases.has_key(scname): # new db dbpath=plugins.makeChannelFilename('%s.db'%scname ,'urldata') UrlReader.databases[scname]=bsddb.btopen(dbpath,'c') urldb = UrlReader.databases[scname] if urldb.has_key(url): poster,title=pickle.loads(urldb[url]) msg='%s has already posted it: %s'%(poster,title) irc.reply(msg.encode('utf-8')) else: try: title=self._getTitle(url) if title != None: urldb[url]=pickle.dumps([nick.decode('utf-8'),title]) urldb.sync() irc.reply(title.encode('utf-8')) except: traceback.print_exc() irc.reply('No Title')
def __init__(self, name, path = "", unique = True): # # Constructs new DiskQueueManager # # @param name : name of the queue # @param path : path where databases will be stored # @param unique : if set to False, disable unique capability, # also add() begin to work exactly as set() # # @param ext_tree : file extention of tree db # @param ext_hash : file extention of hash db ext_tree = ".tree.db" ext_hash = ".hash.db" self.name = name file_md5 = self._hash(name) self.file_tree = path + file_md5 + ext_tree self.file_hash = path + file_md5 + ext_hash self.db_tree = bsddb.btopen( self.file_tree, "c") self.db_hash = bsddb.hashopen(self.file_hash, "c") self.new_key_id = 0L self.unique = unique
def open_bsddb(filename, flag='r', useHash=False, mode=0666): """open bsddb index instead of hash by default. useHash=True forces it to use anydbm default (i.e. hash) instead. Also gives more meaningful error messages.""" try: # 1ST OPEN AS BTREE if useHash: # FORCE IT TO USE HASH INSTEAD OF BTREE return open_anydbm(filename, flag) else: return bsddb.btopen(filename, flag, mode) except bsddb.db.DBAccessError: # HMM, BLOCKED BY PERMISSIONS if flag=='c' or flag=='w': # TRY OPENING READ-ONLY try: ifile = file(filename) except IOError: # HMM, NOT EVEN READABLE. RAISE GENERIC PERMISSIONS ERROR raise PermissionsError('insufficient permissions to open file: ' +filename) ifile.close() # OK, WE CAN READ THE FILE, SO RAISE EXCEPTION WITH raise ReadOnlyError('file is read-only: '+filename) # VERY SPECIFIC MEANING! else: # r OR n FLAG: JUST RAISE EXCEPTION raise PermissionsError('insufficient permissions to open file: ' +filename) except bsddb.db.DBNoSuchFileError: raise NoSuchFileError('no file named: '+filename) except bsddb.db.DBInvalidArgError: # NOT A BTREE FILE... try: if useHash: # NO POINT IN TRYING HASH YET AGAIN... raise bsddb.db.DBInvalidArgError # fallback to using default: hash file return open_anydbm(filename, flag) except bsddb.db.DBInvalidArgError: raise WrongFormatError('file does not match expected shelve format: '+filename)
def main(): try: db = bsddb.btopen(DA_FILE, "w") except: print "DB doesn't exist, creating a new one" db = bsddb.btopen(DA_FILE, "c") lib.set_seed(SEED) for index in range( DB_SIZE): krng = 64 + lib.get_random() % 64 key = str(index) #for i in range(krng): # key += str(unichr(lib.get_random_char())) vrng = 64 + lib.get_random() % 64 value = "" for i in range(vrng): value += str(unichr(lib.get_random_char())) #print key #print value #print "" db[key] = value #try: ##retrieve_key(db) ##retrieve_data(db) ##retrieve_key_range(db) #''' #time1 = time.time()*1000 #if db.has_key('999'): #print db['999'] #time3 = time.time()*1000 - time1 #print time3 #''' #time1 = time.time()*1000 #for i in range(30): #if db.has_key('999'): #print db['999'] #time2 = time.time()*1000 - time1 #print time2, 'millisecaoiands' #''' #print "difference" ,time2/time3 #db.close() #''' #except Exception as e: #print e return db
def deleteDicc(self, tabla_nombre): dicc = db.btopen(self._fileName) if (not tabla_nombre in dicc): dicc.close() return -1 del dicc[tabla_nombre] dicc.close() return 0
def __connect(self): if self.__db is not None: self.close() if self.__option == 'bt': self.__db = bsddb.btopen(self.__filename, self.__flag, self.__mode) if self.__option == 'hash': self.__db = bsddb.hashopen(self.__filename, self.__flag, self.__mode)
def getDiccHeader(self, tabla_nombre): dicc = db.btopen(self._fileName) if (not tabla_nombre in dicc): dicc.close() return [] datos = pickle.loads(dicc[tabla_nombre]) dicc.close() return datos[:-1]
def create_bdb_from_dict(bdb_path, bdb_dict): bdb_path = pathlib2.Path(bdb_path) if bdb_path.exists(): raise nog.exc.MinterError('Path already exists: {}'.format(bdb_path.as_posix())) impl.nog.filesystem.create_missing_directories_for_file(bdb_path) bdb = bsddb.btopen(bdb_path.as_posix(), 'c') bdb.update({bytes(k): bytes(v) for k, v in bdb_dict.items()}) bdb.close()
def __init__bsddb__(self): print "[%s] [INFO] Create the bsddb ..."%(self.__time()) print "[%s] [WARNING] Checking if the bsddb is exist"%(self.__time()) if os.path.exists("pages.db"): # print "[%s] [INFO] Delete the exist db ..."%(self.__time()) os.remove("pages.db") if os.path.exists("visited.db"): # print "[%s] [INFO] Delete the visited.db ..."%(self.__time()) os.remove("visited.db") try: self.pages=bsddb.btopen(file="pages.db",flag='c') self.visited=bsddb.btopen(file="visited.db",flag='c') print "[%s] [INFO] Create db success ..."%(self.__time()) except: print "[%s] [ERROR] Create db error ..."%(self.__time())