示例#1
0
    def compact_db(self):
        """Compacts the DB by deleting any unused space (e.g., after deletion of files)
        """
        assert self._loaded

        if be_verbose:
            print "[luggage.compact_db] Efficiency before: {:.3f}%".format(100 * self.get_db_usage_ratio())

        tmp_optimized_path = self.luggage_path + "_optimized_rnd_{}_{}.raw".format(
            random.randint(0, 10000000), random.randint(0, 10000000)
        )
        tmp_backup_path = self.luggage_path + "_backup_rnd_{}_{}.raw".format(
            random.randint(0, 10000000), random.randint(0, 10000000)
        )
        try:
            shutil.copyfile(self.luggage_path, tmp_backup_path)
            optimized_db = dbm.open(tmp_optimized_path, "c")
            for k, v in self.db.iteritems():
                optimized_db[k] = v
            optimized_db.sync()
            optimized_db.close()
            self.db.close()
            shutil.copyfile(tmp_optimized_path, self.luggage_path)
            self.db = dbm.open(self.luggage_path, "w")
            if be_verbose:
                print "[luggage.compact_db] Efficiency after: {:.3f}%".format(100 * self.get_db_usage_ratio())
        except Exception as ex:
            shutil.copyfile(tmp_backup_path, self.luggage_path)
            raise Exception("[compact_db] Error: {}".format(str(ex)))
        finally:
            for p in [tmp_optimized_path, tmp_backup_path]:
                if os.path.exists(p):
                    os.remove(p)
 def __init__(self, dbname):
     self.__dbname = dbname
     try:
         import dbhash
         dbhash.open(dbname, 'c')
     except:
         shelve.open(dbname)
def _loadDict(request):
    """ Load words from words files or cached dict """
    # check for "dbhash" module
    try:
        import dbhash
    except ImportError:
        dbhash = None

    # load the words
    cachename = os.path.join(request.cfg.data_dir, 'cache', 'spellchecker.dict')
    if dbhash and os.path.exists(cachename):
        wordsdict = dbhash.open(cachename, "r")
    else:
        wordsfiles = _getWordsFiles(request)
        if dbhash:
            wordsdict = dbhash.open(cachename, 'n')
        else:
            wordsdict = {}

        for wordsfile in wordsfiles:
            _loadWordsFile(request, wordsdict, wordsfile)

        if dbhash:
            wordsdict.sync()

    return wordsdict
 def __init__(self, dbname):
     self.__dbname=dbname
     try:
         import dbhash
         dbhash.open(dbname, 'c')
     except:
         shelve.open(dbname)
示例#5
0
def _loadDict(request):
    """ Load words from words files or cached dict """
    # check for "dbhash" module
    try:
        import dbhash
    except ImportError:
        dbhash = None

    # load the words
    cachename = os.path.join(request.cfg.data_dir, 'cache',
                             'spellchecker.dict')
    if dbhash and os.path.exists(cachename):
        wordsdict = dbhash.open(cachename, "r")
    else:
        wordsfiles = _getWordsFiles(request)
        if dbhash:
            wordsdict = dbhash.open(cachename, 'n')
        else:
            wordsdict = {}

        for wordsfile in wordsfiles:
            _loadWordsFile(request, wordsdict, wordsfile)

        if dbhash:
            wordsdict.sync()

    return wordsdict
示例#6
0
def cvtdb(ctx, data, dbtype):
    '''
        Only used for testing purposes
    '''

    db = ctx.parent.params['db']
    newdb = db + '.new'

    if dbtype == 'gdbm':
        import gdbm
        new_d = gdbm.open(newdb, 'n')
    elif dbtype == 'dbm':
        import dbm
        new_d = dbm.open(newdb, 'n')
    elif dbtype == 'dbhash':
        import dbhash
        new_d = dbhash.open(newdb, 'n')
    elif dbtype == 'bsddb':
        new_d = bsddb.hashopen(newdb, 'n')
    elif dbtype == 'dumbdbm':
        import dumbdbm
        new_d = dumbdbm.open(newdb, 'n')
    else:
        raise click.ClickException("Invalid type %s" % dbtype)

    new_data = shelve.Shelf(new_d, protocol=exaile_pickle_protocol)

    for k, v in data.iteritems():
        new_data[k] = v

    new_data.sync()
    new_data.close()
示例#7
0
def load_patch(req,user,did,sid,clear_timeout):
    """ Periodic patch stack read"""
    import time
    base='%s/cg'%__BASE__
    req.content_type = 'text/plain'
    stack = dbhash.open('%s/stack.db'%base,'c')
    t = datetime.datetime.now()
    d = '%s'%time.mktime(t.timetuple())
    stack['_%s'%sid] = '%s:%s'%(d[:10],user)
    if stack.has_key(did):
        if not re.search(sid,stack[did]):
            stack[did] += ':%s'%sid
    else:
        stack[did] = sid
    for other in stack[did].split(':'):
        if (other != sid):
            if stack.has_key('_%s'%other):
                if int(d[:10]) - int(stack['_%s'%other][:10]) > int(clear_timeout):
                    if stack.has_key(did):
                        stack[did] = re.sub(':$','',re.sub('%s:?'%other,'',stack[did]))
                    del stack['_%s'%other]
                    if stack.has_key(other):
                        del stack[other]
    o = ''
    if stack.has_key(sid):
        o = stack[sid]
        del stack[sid]
    stack.close()
    return o
示例#8
0
def get_doc_list(user):
    """ Create a new window id"""
    base='%s/cg'%__BASE__
    o = '<text class="list" onclick="open_doc(evt);" y="20">' 
    if os.path.isfile('%s/stack.db'%base) and os.path.isfile('%s/rev.db'%base):
        rev = dbhash.open('%s/rev.db'%base)
        stack = dbhash.open('%s/stack.db'%base)
        if rev.has_key(user):
            for did in rev[user].split(':'):
                k = '_%s'%did
                name = stack[k].split('\n')[0] if stack.has_key(k) else 'Untitled'
                o += '<tspan x="10" dy="1.2em"><title>%s</title>%s</tspan>'%(did,name)
                o += '<tspan class="small" x="100">1 user</tspan>'
        stack.close()
        rev.close()
    return o + '</text>'
示例#9
0
def cvtdb(ctx, data, dbtype):
    '''
        Only used for testing purposes
    '''
    
    db = ctx.parent.params['db']
    newdb = db + '.new'
    
    if dbtype == 'gdbm':
        import gdbm
        new_d = gdbm.open(newdb, 'n')
    elif dbtype == 'dbm':
        import dbm
        new_d = dbm.open(newdb, 'n')
    elif dbtype == 'dbhash':
        import dbhash
        new_d = dbhash.open(newdb, 'n')
    elif dbtype == 'bsddb':
        new_d = bsddb.hashopen(newdb, 'n')
    elif dbtype == 'dumbdbm':
        import dumbdbm
        new_d = dumbdbm.open(newdb, 'n')
    else:
        raise click.ClickException("Invalid type %s" % dbtype)
    
    new_data = shelve.Shelf(new_d, protocol=exaile_pickle_protocol)
    
    for k, v in data.iteritems():
        new_data[k] = v
    
    new_data.sync()
    new_data.close()
示例#10
0
文件: gitu.py 项目: pelinquin/u
def print_db(db):
    """
    """
    d = dbhash.open(db, 'r')
    for item in d.keys():
        print '[%s] %s -> %s' % (db,item,d[item])
    d.close()
示例#11
0
def print_db(db):
    """
    """
    d = dbhash.open(db, 'r')
    for item in d.keys():
        print '[%s] %s -> %s' % (db, item, d[item])
    d.close()
示例#12
0
def main():
    cache_file = None
    want_ids = 0
    ids = []

    command = None

    for arg in sys.argv[1:]:
        if arg == "-h" or arg == "--help":
            usage()
        elif arg == "-C" or arg == "--channel":
            if command is not None:
                usage_error("Only one command option may be supplied")
            command = "channel"
        elif arg == "-L" or arg == "--list":
            if command is not None:
                usage_error("Only one command option may be supplied")
            command = "list"
        elif arg == "-K" or arg == "--keys":
            if command is not None:
                usage_error("Only one command option may be supplied")
            command = "keys"
        elif arg == "-I" or arg == "--item":
            if command is not None:
                usage_error("Only one command option may be supplied")
            command = "item"
            want_ids = 1
        elif arg == "-H" or arg == "--hide":
            if command is not None:
                usage_error("Only one command option may be supplied")
            command = "hide"
            want_ids = 1
        elif arg == "-U" or arg == "--unhide":
            if command is not None:
                usage_error("Only one command option may be supplied")
            command = "unhide"
            want_ids = 1
        elif arg.startswith("-"):
            usage_error("Unknown option:", arg)
        else:
            if cache_file is None:
                cache_file = arg
            elif want_ids:
                ids.append(arg)
            else:
                usage_error("Unexpected extra argument:", arg)

    if cache_file is None:
        usage_error("Missing expected cache filename")
    elif want_ids and not len(ids):
        usage_error("Missing expected entry ids")

    # Open the cache file directly to get the URL it represents
    try:
        db = dbhash.open(cache_file)
        url = db["url"]
        db.close()
    except dbhash.bsddb._db.DBError, e:
        print >>sys.stderr, cache_file + ":", e.args[1]
        sys.exit(1)
示例#13
0
def SetDev(dev):

    db = dbhash.open(CFG_FILE, 'c')
    db['dev'] = dev
    db.close()

    return True
示例#14
0
def update_tool(req):
    """ update the tool from github """
    import datetime,time,dbhash,re
    from subprocess import Popen, PIPE
    (pwd, name,ip) = get_env(req)
    t = datetime.datetime.now()
    d = time.mktime(t.timetuple())
    rev = dbhash.open('%s/cg/rev.db'%ui.__BASE__,'w')
    allow,delta = False,0
    if rev.has_key('_update_'):
        delta = d - float(rev['_update_'])
        if delta > 120:
            rev['_update_'],allow = '%s'%d,True
    if not rev.has_key('_update_'):
        rev['_update_'] = '%s'%d
    rev.close()
    req.content_type = 'text/plain'  
    if not allow:
        return 'Error: Time since last update is %d secondes; should be greater than 2 minutes!'%int(delta)
    cmd = 'cd %s/..; rm -rf ConnectedGraph; git clone git://github.com/pelinquin/ConnectedGraph.git; cd ConnectedGraph; git submodule update --init'%pwd
    out,err = Popen((cmd), shell=True,stdout=PIPE, stderr=PIPE).communicate()
    o = 'Application Updating from %s %s...\n'%ui.get_id_pkg(req)
    if err:
        o += 'Error:%s\n'%err
    else:
        o += 'Message:%s\nUpdate OK\n'%out
    return o 
示例#15
0
文件: __init__.py 项目: pote/Planet
    def __init__(self, planet, url):
        if not os.path.isdir(planet.cache_directory):
            os.makedirs(planet.cache_directory)
        cache_filename = cache.filename(planet.cache_directory, url)
        cache_file = dbhash.open(cache_filename, "c", 0666)

        cache.CachedInfo.__init__(self, cache_file, url, root=1)

        self._items = {}
        self._planet = planet
        self._expired = []
        self.url = url
        # retain the original URL for error reporting
        self.configured_url = url
        self.url_etag = None
        self.url_status = None
        self.url_modified = None
        self.name = None
        self.updated = None
        self.last_updated = None
        self.filter = None
        self.exclude = None
        self.next_order = "0"
        self.relevant_tags = None
        self.cache_read()
        self.cache_read_entries()

        if planet.config.has_section(url):
            for option in planet.config.options(url):
                value = planet.config.get(url, option)
                self.set_as_string(option, value, cached=0)
示例#16
0
def main():
    print "Pickle is available."
    db = dumbdbm.open("dumbdb", "c")
    db["1"] = "1"
    db.close()
    dbstr = whichdb.whichdb("dumbdb")
    if dbstr:
        print "Dumbdbm is available."
    else:
        print "Dumbdbm is not available."

    db = dbhash.open("dbhash", "c")
    db["1"] = "1"
    db.close()
    dbstr = whichdb.whichdb("dbhash")
    if dbstr == "dbhash":
        print "Dbhash is available."
    else:
        print "Dbhash is not available."

    if bsddb is None:
        dbstr = ""
    else:
        db = bsddb.hashopen("bsddb3", "c")
        db["1"] = "1"
        db.close()
        dbstr = whichdb.whichdb("bsddb3")
    if dbstr == "dbhash":
        print "Bsddb[3] is available."
    else:
        print "Bsddb[3] is not available."

    print

    hammie = get_pathname_option("Storage", "persistent_storage_file")
    use_dbm = options["Storage", "persistent_use_database"]
    if not use_dbm:
        print "Your storage %s is a: pickle" % (hammie,)
        return

    if not os.path.exists(hammie):
        print "Your storage file does not exist yet."
        return
    db_type = whichdb.whichdb(hammie)
    if db_type == "dbhash":
        # could be dbhash or bsddb3
        # only bsddb3 has a __version__ attribute - old bsddb module does not
        if hasattr(bsddb, '__version__'):
            try:
                db = bsddb.hashopen(hammie, "r")
            except bsddb.error:
                pass
            else:
                db.close()
                print "Your storage", hammie, "is a: bsddb[3]"
                return
    elif db_type is None:
        print "Your storage %s is unreadable." % (hammie,)
    print "Your storage %s is a: %s" % (hammie, db_type)
示例#17
0
def main():
    print "Pickle is available."
    db = dumbdbm.open("dumbdb", "c")
    db["1"] = "1"
    db.close()
    dbstr = whichdb.whichdb("dumbdb")
    if dbstr:
        print "Dumbdbm is available."
    else:
        print "Dumbdbm is not available."

    db = dbhash.open("dbhash", "c")
    db["1"] = "1"
    db.close()
    dbstr = whichdb.whichdb("dbhash")
    if dbstr == "dbhash":
        print "Dbhash is available."
    else:
        print "Dbhash is not available."

    if bsddb is None:
        dbstr = ""
    else:
        db = bsddb.hashopen("bsddb3", "c")
        db["1"] = "1"
        db.close()
        dbstr = whichdb.whichdb("bsddb3")
    if dbstr == "dbhash":
        print "Bsddb[3] is available."
    else:
        print "Bsddb[3] is not available."

    print

    hammie = get_pathname_option("Storage", "persistent_storage_file")
    use_dbm = options["Storage", "persistent_use_database"]
    if not use_dbm:
        print "Your storage %s is a: pickle" % (hammie,)
        return

    if not os.path.exists(hammie):
        print "Your storage file does not exist yet."
        return
    db_type = whichdb.whichdb(hammie)
    if db_type == "dbhash":
        # could be dbhash or bsddb3
        # only bsddb3 has a __version__ attribute - old bsddb module does not
        if hasattr(bsddb, '__version__'):
            try:
                db = bsddb.hashopen(hammie, "r")
            except bsddb.error:
                pass
            else:
                db.close()
                print "Your storage", hammie, "is a: bsddb[3]"
                return
    elif db_type is None:
        print "Your storage %s is unreadable." % (hammie,)
    print "Your storage %s is a: %s" % (hammie, db_type)
示例#18
0
 def __init__(self, filename, flag='c'):
     self.filename = filename
     self.db = dbhash.open(self.filename, flag)
     self.max_id = int(self.db.get('M_max_id', 0))
     self.sort_by = self.sortByPositionAndFrequency
     # TODO: consider requirement for loading stopwords here
     # or speed it up
     self.stopwords = [line.strip() for line in open(dolphy.STOPWORDS)]
示例#19
0
def create_window_id():
    """ Create a new window id"""
    base='%s/cg'%__BASE__
    rev = dbhash.open('%s/rev.db'%base,'c')
    rev['@'] = '%d'%(long(rev['@'])+1) if rev.has_key('@') else '0'
    sid = hashlib.sha1(rev['@']).hexdigest()[:-20]
    rev.close()    
    return sid
示例#20
0
文件: index.py 项目: tomdyson/dolphy
	def __init__(self, filename, flag='c'):
		self.filename = filename
		self.db = dbhash.open(self.filename, flag)
		self.max_id = int(self.db.get('M_max_id', 0))
		self.sort_by = self.sortByPositionAndFrequency
		# TODO: consider requirement for loading stopwords here
		# or speed it up
		self.stopwords = [line.strip() for line in open(dolphy.STOPWORDS)]
示例#21
0
def get_new_obj(dbpart):
    traceroutetable = config['BDM_BSD_FILE']
    try:
        obj = db.open('%s-%s.db' % (traceroutetable, dbpart), 'w')
        print 'opening %s-%s.db' % (traceroutetable, dbpart)
        return obj
    except:
        return None
def get_new_obj(dbpart):
  traceroutetable = config['BDM_BSD_FILE']
  try:
    obj = db.open('%s-%s.db'%(traceroutetable,dbpart),'w')
    print 'opening %s-%s.db'%(traceroutetable,dbpart)
    return obj
  except:
    return None
示例#23
0
def GetDev():
    db = dbhash.open(CFG_FILE, 'c')
    if db.has_key('dev') == False:
        db['dev'] = u''

    dev = db['dev']
    db.close()

    return dev
示例#24
0
def cacheEntries(cachefile=dnscachefile):
  db=dbhash.open(cachefile,"r")
  for key, value in dbseq(db):
    hostlen=len(value)-12
    when, numeric, foo, name = struct.unpack("III%ds"%hostlen, value)
    name=name.rstrip('\x00')
    if not numeric:
      yield key, when, name
  db.close()
示例#25
0
def save_doc(user,did,title,lout,content):
    """ Save the layout and the content of a diagram """
    base = __BASE__
    stack = dbhash.open('%s/stack.db'%base,'c')
    stack['_%s'%did] = title + '\n' + lout + '\n' + content
    k = '@%s'%did
    if stack.has_key(k):
        del stack[k]
    stack.close()
    return 'Saved'
示例#26
0
def check_user(login,pw):
    """ check if user registered with good password """
    result, base = False, '%s/pw.db'%__BASE__
    if login and pw:
        if os.path.isfile(base):
            db = dbhash.open(base)
            if db.has_key(login):
                if db[login] == hashlib.sha1(login+pw).hexdigest():
                    result = True
            db.close()    
    return result
示例#27
0
def open():
    try:
        cache = config.cache_directory()
        index=os.path.join(cache,'index')
        if not os.path.exists(index): return None
        import dbhash
        return dbhash.open(filename(index, 'id'),'w')
    except Exception, e:
        if e.__class__.__name__ == 'DBError': e = e.args[-1]
        from planet import logger as log
        log.error(str(e))
示例#28
0
def change_pw_user(login,pw,pw2):
    """ Change password for a registered user"""
    result, base = False, '%s/pw.db'%__BASE__
    if login and pw and pw2 and os.path.isfile(base):
        db = dbhash.open(base,'c')
        if db.has_key(login):
            if db[login] == hashlib.sha1(login+pw).hexdigest():
                if (len(pw2) > 4) and (pw2 != pw) and (pw2 != login):
                    db[login],result = hashlib.sha1(login+pw2).hexdigest(),True
        db.close() 
    return result
示例#29
0
def get_buddyicon(flickr, data, size=48):
    """Lookup the buddyicon from the data in @data using @flickr and resize it
    to @size pixels."""
    from twisted.web.client import getPage
    import dbhash, bsddb

    global __buddy_cache
    if __buddy_cache is None:
        folder = os.path.join(get_cache_path(), "postr")
        if not os.path.exists(folder):
            os.makedirs(folder)
        path = os.path.join(folder, "buddyicons")
        try:
            __buddy_cache = dbhash.open(path, "c")
        except bsddb.db.DBInvalidArgError:
            # The database needs upgrading, so delete it
            os.remove(path)
            __buddy_cache = dbhash.open(path, "c")

    def load_thumb(page, size):
        loader = gtk.gdk.PixbufLoader()
        loader.set_size(size, size)
        loader.write(page)
        loader.close()
        return loader.get_pixbuf()

    def got_data(page, url, size):
        __buddy_cache[url] = page
        return load_thumb(page, size)

    if int(data.get("iconfarm")) > 0:
        url = "http://farm%s.static.flickr.com/%s/buddyicons/%s.jpg" % (
            data.get("iconfarm"), data.get("iconserver"), data.get("nsid"))
    else:
        url = "http://www.flickr.com/images/buddyicon.jpg"

    if __buddy_cache.has_key(url):
        from twisted.internet import defer
        return defer.succeed(load_thumb(__buddy_cache[url], size))
    else:
        return getPage(url).addCallback(got_data, url, size)
示例#30
0
def new_doc(req,user):
    """ utiliser user """
    base='%s/cg'%__BASE__
    req.content_type = 'text/plain'
    rev = dbhash.open('%s/rev.db'%base,'c')
    gid = create_id(rev)
    if rev.has_key(user):
        rev[user] += ':%s'%gid
    else:
        rev[user] = gid
    rev.close()    
    return gid
示例#31
0
def check_user(login,pw):
    """ """
    base='%s/cg/pw.db'%__BASE__
    result = False
    if login and pw:
        if os.path.isfile(base):
            db = dbhash.open(base)
            if db.has_key(login):
                if db[login] == hashlib.sha1(pw).hexdigest():
                    result = True
            db.close()    
    return result
示例#32
0
文件: util.py 项目: frojasg/postr
def get_buddyicon(flickr, data, size=48):
    """Lookup the buddyicon from the data in @data using @flickr and resize it
    to @size pixels."""
    from twisted.web.client import getPage
    import dbhash, bsddb

    global __buddy_cache
    if __buddy_cache is None:
        folder = os.path.join (get_cache_path(), "postr")
        if not os.path.exists(folder):
            os.makedirs(folder)
        path = os.path.join (folder, "buddyicons")
        try:
            __buddy_cache = dbhash.open(path, "c")
        except bsddb.db.DBInvalidArgError:
            # The database needs upgrading, so delete it
            os.remove(path)
            __buddy_cache = dbhash.open(path, "c")
    
    def load_thumb(page, size):
        loader = gtk.gdk.PixbufLoader()
        loader.set_size (size, size)
        loader.write(page)
        loader.close()
        return loader.get_pixbuf()

    def got_data(page, url, size):
        __buddy_cache[url] = page
        return load_thumb(page, size)
    
    if int(data.get("iconfarm")) > 0:
        url = "http://farm%s.static.flickr.com/%s/buddyicons/%s.jpg" % (data.get("iconfarm"), data.get("iconserver"), data.get("nsid"))
    else:
        url = "http://www.flickr.com/images/buddyicon.jpg"

    if __buddy_cache.has_key(url):
        from twisted.internet import defer
        return defer.succeed(load_thumb(__buddy_cache[url], size))
    else:
        return getPage(url).addCallback(got_data, url, size)
示例#33
0
def new_doc(user):
    """ utiliser user """
    gid = ''
    if user:
        base = __BASE__
        rev = dbhash.open('%s/rev.db'%base,'c')
        gid = create_id(rev)
        if rev.has_key(user):
            rev[user] += ':%s'%gid
        else:
            rev[user] = gid
        rev.close()    
    return gid
示例#34
0
def register(content=''):
    """ If the same content is requested, then id does not change """
    base = __BASE__
    if not os.path.isdir(base):
        os.mkdir(base)
    rev = dbhash.open('%s/rev.db' % base, 'c')
    if rev.has_key(content):
        gid = rev[content]
    else:
        gid = create_id(rev)
        rev[content] = gid
    rev.close()
    return gid
示例#35
0
文件: gitu.py 项目: pelinquin/u
def register(content=''):
    """ If the same content is requested, then id does not change """
    base = __BASE__
    if not os.path.isdir(base):
        os.mkdir(base)
    rev = dbhash.open('%s/rev.db'%base,'c')
    if rev.has_key(content):
        gid = rev[content]
    else:
        gid = create_id(rev)
        rev[content] = gid
    rev.close()    
    return gid
示例#36
0
文件: idindex.py 项目: xxjeng/nuxleus
def open():
    try:
        cache = config.cache_directory()
        index = os.path.join(cache, "index")
        if not os.path.exists(index):
            return None
        return dbhash.open(filename(index, "id"), "w")
    except Exception, e:
        if e.__class__.__name__ == "DBError":
            e = e.args[-1]
        from planet import logger as log

        log.error(str(e))
示例#37
0
def dbhash_sample():
	#打开一个文件,假如不存在则创建一个。
	db = dbhash.open('dbhash.dat', 'c')
	db['Tom'] = 'Beijing Road'
	db['Jerry'] = 'Shanghai Road'
	# db['Jim'] = 13 #出错,因为在dbhash模块中要求字典的键和值都必须是字符串。
	for k, v in db.iteritems():
		print k, v

	if db.has_key('Tom'):
		del db['Tom']
	print db
	db.close()
示例#38
0
def feed(url):
    fn = base64.b64encode(url)
    fn = os.path.join(temp, fn)
    now = datetime.date.today()
    d = kvdb.open(os.path.join(temp, 'update'), 'c')
    if d.has_key(url) and d[url] == str(now):
        return fn

    with open(fn, 'wb') as f:
        rep = urllib.urlopen(url)
        f.write(rep.read())
    d[url] = str(now)
    return fn
示例#39
0
    def create_new_luggage(luggage_path, password, pyluggage_config):
        """Create a new Luggage file and open it, returning an open instance
        """
        # TODO: pyluggage_config -> apply config to LuggageParams call
        if os.path.exists(luggage_path):
            os.remove(luggage_path)
        params = LuggageParams()
        db = dbm.open(luggage_path, "c")
        db[Luggage.params_db_name] = params.dumps()
        db.close()

        return Luggage(
            luggage_path=luggage_path, password=password, pyluggage_config=pyluggage_config, is_creating=True
        )
示例#40
0
def export_day(day, conn):
    db = dbhash.open("traceroutes--2014-01.db", "r")

    for keys in db.keys():
        key = cp.loads(keys)
        value = cp.loads(db[keys])
        retVal = run_insert_query_by_day(conn, key, value, day)
        # if there is an error or we have gone past the correct day,
        # then break out
        # if retVal == False:
        #     break

    conn.commit()
    db.close()
示例#41
0
def main():
    """Main function"""

    # WRITE #######

    db = dbhash.open('foo_dbhash', 'c')

    db['one'] = 'un'
    db['two'] = 'dos'
    db['three'] = 'tres'

    db.close()

    # WHICH DBM ###

    print "whichdb:", whichdb.whichdb('foo_dbhash')
    print 

    # READ ########

    db = dbhash.open('foo_dbhash', 'r')

    # Iterate loop: first method (common to any dbm module)
    for k in db.keys():
        print k, ':', db[k]

    # Iterate loop: second method (dbhash supports db.items())
    for k, v in db.items():
        print k, ':', v

    # Iterate loop: third method (the most efficient)
    # The following code prints every key in the database db, without having to create a list in memory that contains them all.
    print db.first()
    for i in xrange(1, len(db)):
        print db.next()

    db.close()
示例#42
0
def main():
    """Main function"""

    # WRITE #######

    db = dbhash.open('foo_dbhash', 'c')

    db['one'] = 'un'
    db['two'] = 'dos'
    db['three'] = 'tres'

    db.close()

    # WHICH DBM ###

    print "whichdb:", whichdb.whichdb('foo_dbhash')
    print

    # READ ########

    db = dbhash.open('foo_dbhash', 'r')

    # Iterate loop: first method (common to any dbm module)
    for k in db.keys():
        print k, ':', db[k]

    # Iterate loop: second method (dbhash supports db.items())
    for k, v in db.items():
        print k, ':', v

    # Iterate loop: third method (the most efficient)
    # The following code prints every key in the database db, without having to create a list in memory that contains them all.
    print db.first()
    for i in xrange(1, len(db)):
        print db.next()

    db.close()
示例#43
0
def export_all(conn):
    db = dbhash.open("traceroutes--2014-01.db", "r")
    numAdded = 0

    for keys in db.keys():
        key = cp.loads(keys)
        value = cp.loads(db[keys])
        retVal = run_insert_query(conn, key, value)
        numAdded += 1
        if numAdded > 100:
            conn.commit()
            numAdded = 0

    conn.commit()
    db.close()
示例#44
0
def create():
    from planet import logger as log
    cache = config.cache_directory()
    index=os.path.join(cache,'index')
    if not os.path.exists(index): os.makedirs(index)
    import dbhash
    index = dbhash.open(filename(index, 'id'),'c')

    try:
        import libxml2
    except:
        libxml2 = False
        from xml.dom import minidom

    for file in glob(cache+"/*"):
        if os.path.isdir(file):
            continue
        elif libxml2:
            try:
                doc = libxml2.parseFile(file)
                ctxt = doc.xpathNewContext()
                ctxt.xpathRegisterNs('atom','http://www.w3.org/2005/Atom')
                entry = ctxt.xpathEval('/atom:entry/atom:id')
                source = ctxt.xpathEval('/atom:entry/atom:source/atom:id')
                if entry and source:
                    index[filename('',entry[0].content)] = source[0].content
                doc.freeDoc()
            except:
                log.error(file)
        else:
            try:
                doc = minidom.parse(file)
                doc.normalize()
                ids = doc.getElementsByTagName('id')
                entry = [e for e in ids if e.parentNode.nodeName == 'entry']
                source = [e for e in ids if e.parentNode.nodeName == 'source']
                if entry and source:
                    index[filename('',entry[0].childNodes[0].nodeValue)] = \
                        source[0].childNodes[0].nodeValue
                doc.freeDoc()
            except:
                log.error(file)

    log.info(str(len(index.keys())) + " entries indexed")
    index.close()

    return open()
示例#45
0
#!/usr/local/bin/python2.5
import web, dbhash, sys
import simplejson as json
from openlibrary.catalog.load import add_keys
from copy import deepcopy
from openlibrary.catalog.merge.index import *
from urllib import urlopen, urlencode

path = '/1/edward/marc_index/'
#dbm_fields = ('lccn', 'oclc', 'isbn', 'title')
#dbm = dict((i, dbhash.open(path + i + '.dbm', flag='w')) for i in dbm_fields)

store_db = dbhash.open(path + "store.dbm", flag='w')

urls = (
#    '/', 'index',
    '/store/(.*)', 'store',
    '/keys', 'keys',
)

app = web.application(urls, globals())

def build_pool(index_fields): # unused
    pool = {}
    for field in dbm_fields:
        if not field in index_fields:
            continue
        for v in index_fields[field]:
            if field == 'isbn' and len(v) < 10:
                continue
            try:
import dbhash
db = dbhash.open("dbhash", "c")
db["one"] = "the foot"
db["two"] = "the shoulder"
db["three"] = "the other foot"
db["four"] = "the bridge of the nose"
db["five"] = "the naughty bits"
db["six"] = "just above the elbow"
db["seven"] = "two inches to the right of a very naughty bit indeed"
db["eight"] = "the kneecap"
db.close()
db = dbhash.open("dbhash", "r")
for key in db.keys():
    print repr(key), repr(db[key])
示例#47
0
import os, re, sys, codecs, dbhash
from catalog.amazon.other_editions import find_others
from catalog.infostore import get_site
from catalog.read_rc import read_rc
from catalog.get_ia import get_data
from catalog.marc.build_record import build_record
from catalog.marc.fast_parse import get_tag_lines, get_all_subfields

sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
rc = read_rc()
db = dbhash.open(rc['index_path'] + 'isbn_to_marc.dbm', 'r')

site = get_site()


def get_records_from_marc(isbn):
    if isbn not in db:
        return


#    for loc in db[isbn].split(' '):
#        data = get_data(loc)
#        print loc
#        want = ['100', '110', '111', '240', '245', '260'] + [str(i) for i in range(500,600) if i not in (505, 520)]
#        for tag, line in get_tag_lines(data, set(want)):
#            sub = list(get_all_subfields(line))
#            if tag.startswith('5'):
#                assert len(sub) == 1 and sub[0][0] == 'a'
#                note = sub[0][1]
#                if note.find('ublish') != -1 or note.find('riginal') != -1:
#                    print '  note:', note
示例#48
0
from __future__ import print_function
import web
import dbhash

dbm = dbhash.open("/2/pharos/imagepdfs/oclc_to_marc.dbm", "r")

urls = ('/', 'index')


class index:
    def GET(self):
        web.header('Content-Type', 'text/html; charset=utf-8', unique=True)

        input = web.input()
        if 'oclc' in input:
            html_oclc = web.htmlquote(input.oclc)
            print("<html>\n<head><title>OCLC to MARC: %s</title><body>" %
                  html_oclc)
        else:
            print("<html>\n<head><title>OCLC to MARC</title><body>")
        print('<form method="get">')
        print('OCLC:')
        if 'oclc' in input:
            print('<input type="text" name="oclc" value="%s">' % html_oclc)
        else:
            print('<input type="text" name="oclc">')
        print('<input type="submit" value="Find MARC">')
        print('</form>')

        if 'oclc' in input:
            print('Searching for OCLC: %s<p>' % html_oclc)
示例#49
0
 def database(self):
     import dbhash
     if self.__database is None:
         dbname = os.path.abspath(os.path.splitext(sys.argv[0])[0] + '.db')
         self.__database = dbhash.open(dbname, 'w')
     return self.__database
示例#50
0
        return ''


def insert_value(id, key, val):
    return "INSERT INTO contacts VALUES (" + str(
        id) + ",'" + key + "',\"" + val + "\");\n"


file = open("/tmp/contacts.sql", "w")
file.write("drop table contacts;\n")
file.write("drop table contacts_urn;\n")
file.write(
    "create table contacts (urn INTEGER NOT NULL, tag TEXT NOT NULL, value TEXT NOT NULL);\n"
)
file.write("create table contacts_urn (urn INTEGER PRIMARY KEY);\n")
db = dbhash.open(
    os.environ['HOME'] + "/evolution/local/Contacts/addressbook.db", "r")

x = 0
z = len(db.keys())
db.first()
while x < z - 1:
    vcard_key = db.next()
    vcard_data = open_vcard(vcard_key[1])
    if vcard_data != '':
        file.write("insert into contacts_urn values (" + str(x + 1) + ");\n")
        file.write(
            insert_value(x + 1, "NAME", get_vcard_data(vcard_data, "FN:")))
        file.write(
            insert_value(x + 1, "HOME.EMAIL",
                         get_vcard_data(vcard_data, "EMAIL;INTERNET:")))
        file.write(
示例#51
0
# test performance of serialisers and persisted hashes for dolphy
# TODO: try shelve (should be same as dbhash and cPickle)
# TODO: try low level bsdbb driver
# TODO: use more realistic hash
# TODO: compare index filesizes

import dbhash
import time
import marshal
import cPickle as pickle

DBFILE = 'test.db'
REPITITIONS = 1000

db = dbhash.open(DBFILE, flag='c')

print "=== Starting Test with %s repititions ===" % REPITITIONS
# using eval
d = "{'a':['one','two','three'],'b':[3,2,1], 'c':['this is a longer string']}"

start = time.time()

for i in range(0, REPITITIONS):
    db[str(i)] = d

print "populated REPITITIONS dicts as strings in %.3f seconds" % (time.time() -
                                                                  start)

start = time.time()

for i in range(0, REPITITIONS):
示例#52
0
def run(args):
    import config

    if sys.version_info < (2, 5):
        sys.path.append(os.path.join(config.progdir, 'py25modules'))

    parser = BakefileOptionParser()
    parser.add_option('-f',
                      '--format',
                      action="store",
                      dest='format',
                      help='format of generated makefile')
    parser.add_option('-o',
                      '--output',
                      action="store",
                      dest='outfile',
                      help='output file')
    parser.add_option('-D',
                      action="append",
                      dest='defines',
                      metavar='VAR=VALUE',
                      help='override variable or option definition')
    parser.add_option('-I',
                      action="append",
                      dest='includes',
                      metavar='PATH',
                      help='search for bakefile rules in PATH')
    parser.add_option('-v',
                      '--verbose',
                      action="store_true",
                      dest='verbose',
                      default=0,
                      help='display detailed information')
    parser.add_option('-q',
                      '--quiet',
                      action="store_true",
                      dest='quiet',
                      default=0,
                      help='supress all output except of errors')
    parser.add_option('',
                      '--dry-run',
                      action="store_true",
                      dest='dry_run',
                      default=0,
                      help="don't write any files, just pretend doing it")
    parser.add_option(
        '',
        '--touch',
        action="store_true",
        dest='always_touch_output',
        default=0,
        help="always touch output files, even if their content doesn't change")
    parser.add_option(
        '',
        '--eol',
        default="format",
        action="store",
        dest='eol',
        metavar='STYLE',
        type='choice',
        choices=['format', 'dos', 'unix', 'mac', 'native'],
        help=
        "line endings type to use in output files (format, dos, unix, mac, native) [default: format]"
    )
    parser.add_option(
        '',
        '--wrap-output',
        default="75",
        action="store",
        dest='wrap_lines_at',
        metavar='LENGTH',
        help=
        "don't generate lines longer than LENGTH; set to \"no\" to disable wrapping [default: 75]"
    )
    parser.add_option('',
                      '--output-deps',
                      action="store",
                      dest='deps_file',
                      metavar='DEPSFILE',
                      help="output dependencies information for bakefile_gen")
    parser.add_option('',
                      '--output-changes',
                      action="store",
                      dest='changes_file',
                      metavar='MODSFILE',
                      help="output list of modified files to a file")
    parser.add_option(
        '',
        '--xml-cache',
        action="store",
        dest='xml_cache',
        metavar='CACHEFILE',
        help="cache file where bakefile_gen stores pre-parsed XML files")
    parser.add_option('',
                      '--debug',
                      action="store_true",
                      dest='debug',
                      default=0,
                      help="show debugging information (you don't want this)")
    parser.add_option('',
                      '--dump',
                      action="store_true",
                      dest='dump',
                      default=0,
                      help='dump parsed makefile content instead of ' +
                      'generating makefile')

    options, args = parser.parse_args(args)

    if len(args) != 1:
        parser.error('incorrect number of arguments, exactly 1 .bkl required')

    config.dry_run = options.dry_run
    config.always_touch_output = options.always_touch_output
    config.debug = options.debug
    config.quiet = options.quiet
    if config.quiet:
        config.verbose = 0
    else:
        config.verbose = options.verbose

    if options.includes != None:
        addIncludePaths(options.includes)

    if options.xml_cache != None:
        try:
            # don't use shelve.open(), limit ourselves to dbhash version by
            # using the Shelf class explicitly; that's because we store large
            # chunks of data in the shelve and some dbm implementations can't
            # handle it -- see http://www.bakefile.org/ticket/214
            import pickle, shelve, dbhash
            xmlparser.cache = shelve.Shelf(dbhash.open(options.xml_cache, 'c'),
                                           protocol=pickle.HIGHEST_PROTOCOL)
        except ImportError:
            sys.stderr.write(
                "Warning: disabling XML cache because it's not supported by this version of Python\n"
            )
        except AttributeError:  # python < 2.3 didn't have HIGHEST_PROTOCOL
            sys.stderr.write(
                "Warning: disabling XML cache because it's not supported by Python 2.2\n"
            )
        except KeyError:  # python < 2.3 didn't have protocol argument
            sys.stderr.write(
                "Warning: disabling XML cache because it's not supported by Python 2.2\n"
            )
        except Exception, e:
            sys.stderr.write(
                "Warning: disabling XML cache because an error occured while loading %s:\n         %s\n"
                % (options.xml_cache, e))
示例#53
0
        else:
            if cache_file is None:
                cache_file = arg
            elif want_ids:
                ids.append(arg)
            else:
                usage_error("Unexpected extra argument:", arg)

    if cache_file is None:
        usage_error("Missing expected cache filename")
    elif want_ids and not len(ids):
        usage_error("Missing expected entry ids")

    # Open the cache file directly to get the URL it represents
    try:
        db = dbhash.open(cache_file)
        url = db["url"]
        db.close()
    except dbhash.bsddb._db.DBError, e:
        print >> sys.stderr, cache_file + ":", e.args[1]
        sys.exit(1)
    except KeyError:
        print >> sys.stderr, cache_file + ": Probably not a cache file"
        sys.exit(1)

    # Now do it the right way :-)
    my_planet = planet.Planet(ConfigParser.ConfigParser())
    my_planet.cache_directory = os.path.dirname(cache_file)
    channel = planet.Channel(my_planet, url)

    for item_id in ids:
示例#54
0
import dbhash
db = dbhash.open('dbhashtemp','c')
db['西施'] = '西施浣纱'
db['貂蝉'] = '貂蝉拜月'
db['昭君'] = '昭君出塞'
print "没有进行任何操作:"
for k,v in db.iteritems():
    print k,v
if db.has_key('西施'):
    del db['西施']
print "删除西施对应的数据:"
for k,v in db.iteritems():
    print k,v  
db.close()
    
示例#55
0
# -*- coding:utf-8 -*-
author = 'haibo'
time = '2018-7-28'
"""
数据持久化

dbhash 只支持字符串类型的值
shelve 返回的字典值,支持:字符串、数字、元祖以及列表
"""

import dbhash

db = dbhash.open('tem', 'c')
db['西施'] = '西施浣纱'
db['貂蝉'] = '貂蝉拜月'
db['昭君'] = '昭君出塞'

print '--------------没有进行任何操作的数据---------------'
for k, v in db.iteritems():
    print k, v
if db.has_key('西施'):
    del db['西施']
print '------------删除键为‘西施’对应的数据---------------'
for k, v in db.iteritems():
    print k, v
db.close()
示例#56
0
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import dbhash

db = dbhash.open('temp', 'c')               # 创建并打开数据库
db['Tom'] = 'Beijing road'                  # 写数据
db['Jerry'] = 'Shanghai road'
for k, v in db.iteritems():                 # 遍历db对象
    print k, v
if db.has_key('Tom'):
    del db['Tom']
db['Tod'] = 2                               # 非字符串的键或值会引发异常
print db
db.close()                                  # 关闭数据库
示例#57
0
outname = None
dbs = []
for arg in sys.argv[1:]:
    if 'outdb' in arg:
        outname = arg.split('=')[1]
    else:
        if os.path.exists(arg):
            dbs.append(arg)
            print('FOUND: %s' % arg)
        else:
            print('WARNING: %s was specified but not found' % arg)

if not outname:
    raise Exception('must specify outdb=filename')

ndb = dbhash.open(outname, 'n')

outerdb = {}
for db in dbs:
    print('Merging... %s' % db)
    opendb = dbhash.open(db, 'r')
    keys = opendb.keys()
    for key in keys:
        print('...Key: %s' % key)
        outerdb.setdefault(key, {})
        objects = pickle.loads(opendb[key])
        for obj in objects.keys():
            print('......Subkey: %s' % obj)
            if outerdb[key].has_key(obj):
                print('WARNING: key collision: %s dbase key: %s ' % (db, obj))
            else: