def show_public_timeline(since_ids = None, hashtag_list = None, hashtag_db_name = None, tweet_db_name = None): ''' user - twitter.User object since_ids - twitter.Status.id (the timeline will consist of all the tweets after the tweet with id as since_ids) hashtag_list - A list of hashtags if you want to search for particular hashtags hashtag_db_name - Provide a string name, If you want to save the hashtags and the counts. It will be stored in a gdbm file. tweet_db_name - Provide a string name, If you want to save the tweets hashtags and the tweet id. It will be stored in a gdbm file. Returns the tweet id of the latest tweet. ''' timeline = api.GetPublicTimeline(since_id = since_ids) if not timeline: return since_ids hashtag_public_db = None tweet_db = None if hashtag_db_name: hashtag_public_db = gdbm.open(hashtag_db_name,'c') if tweet_db_name: tweet_db = gdbm.open(tweet_db_name,'c') since_ids = show_timeline(timeline, hashtag_list = hashtag_list, hashtag_db = hashtag_public_db, tweet_db = tweet_db) if hashtag_db_name: hashtag_public_db.close() if tweet_db_name: tweet_db.close() return since_ids
def main(): """Main function""" # WRITE ####### db = gdbm.open('foo_gdbm', 'c') db['one'] = 'un' db['two'] = 'dos' db['three'] = 'tres' db.close() # WHICH DBM ### print "whichdb:", whichdb.whichdb('foo_gdbm') print # READ ######## db = gdbm.open('foo_gdbm', 'r') # Iterate loop: first method (common to any dbm module) for k in db.keys(): print k, ':', db[k] # Iterate loop: second method (more efficient) # The following code prints every key in the database db, without having to create a list in memory that contains them all. k = db.firstkey() while k != None: print k, ':', db[k] k = db.nextkey(k) db.close()
def setproxy(command): proxystr = command.strip("--proxy ") proxystr.strip("\r\n") print proxystr, if proxystr.startswith("http:"): f = gdbm.open("proxyfile","c") f["http"] = proxystr f.close() elif proxystr.startswith("https:"): f = gdbm.open("proxyfile","c") f["https"] = proxystr f.close() elif proxystr is "None" or proxystr is "none": f = gdbm.open("proxyfile","c") f["https"] = "" f["http"] = "" print f["https"] print f["http"] f.close() print "proxy set to none" else: f = gdbm.open("proxyfile","c") f["https"] = "" f["http"] = "" print f["https"] print f["http"] f.close() return proxystr
def __init__(self, dbname): try: import gdbm gdbm.open(dbname) self.__db = shelve.DbfilenameShelf(dbname) except: self.__db = shelve.open(dbname)
def __init__(self, settings, datadir, log, mempool, netmagic, readonly=False, fast_dbm=False): self.settings = settings self.log = log self.mempool = mempool self.readonly = readonly self.netmagic = netmagic self.fast_dbm = fast_dbm self.blk_cache = Cache(750) self.orphans = {} self.orphan_deps = {} if readonly: mode_str = 'r' else: mode_str = 'c' if fast_dbm: self.log.write("Opening database in fast mode") mode_str += 'f' self.misc = gdbm.open(datadir + '/misc.dat', mode_str) self.blocks = gdbm.open(datadir + '/blocks.dat', mode_str) self.height = gdbm.open(datadir + '/height.dat', mode_str) self.blkmeta = gdbm.open(datadir + '/blkmeta.dat', mode_str) self.tx = gdbm.open(datadir + '/tx.dat', mode_str) if 'height' not in self.misc: self.log.write("INITIALIZING EMPTY BLOCKCHAIN DATABASE") self.misc['height'] = str(-1) self.misc['msg_start'] = self.netmagic.msg_start self.misc['tophash'] = ser_uint256(0L) self.misc['total_work'] = hex(0L) if 'msg_start' not in self.misc or (self.misc['msg_start'] != self.netmagic.msg_start): self.log.write("Database magic number mismatch. Data corruption or incorrect network?") raise RuntimeError
def __init__(self): self.id_idx_db = gdbm.open('blockid_idx.db', 'cf') self.num_idx_db = gdbm.open('blocknum_idx.db', 'cf') self.blockdata_db = gdbm.open('blockdata.db', 'cf') self.party_idxa_db = gdbm.open('party_idx_a.db', 'cf') self.party_idxb_db = gdbm.open('party_idx_b.db', 'cf') self.mc = memcache.Client(config.mc_endpoint)
def saveuserdata(self,mainclass): iter=mainclass.elementy.get_iter(0) i=False if mainclass.combo1.child.get_text()!="": while iter: if mainclass.combo1.child.get_text()!= mainclass.elementy.get_value(iter,0): i=True else: i=False break; iter=mainclass.elementy.iter_next(iter) if i: d = gdbm.open(userdata, 'c') l=len(d.keys()) d[str(l)]=str(mainclass.combo1.child.get_text()) d.close() i=False iter=mainclass.elementy2.get_iter(0) if mainclass.combo2.child.get_text()!="": while iter: if mainclass.combo2.child.get_text()!= mainclass.elementy2.get_value(iter,0): i=True else: i=False break; iter=mainclass.elementy2.iter_next(iter) if i: d = gdbm.open(userdata2, 'c') l=len(d.keys()) d[str(l)]=str(mainclass.combo2.child.get_text()) d.close()
def get(self): session_store = sessions.get_store(request=self.request) session = session_store.get_session() username = session.get('username') key = session.get('key') try: database = gdbm.open('database.gdbm','cf') except: time.sleep(1.0) database = gdbm.open('database.gdbm','cf') if username: if ('usr'+username).encode('ascii','ignore') in database: auth_info = json.loads( database[('usr'+username).encode('ascii','ignore')] ) if auth_info['key'] == key: template_values = {'auth':auth_info} template = JINJA_ENVIRONMENT.get_template('ko.html') database.close() self.response.write(template.render(template_values)) else: database.close() self.redirect('/jj70/login') else: database.close() self.redirect('/jj70/login') else: database.close() self.redirect('/jj70/login')
def post(self): username = self.request.get('username') password = self.request.get('password') try: database = gdbm.open('database.gdbm','cf') except: time.sleep(0.1) database = gdbm.open('database.gdbm','cf') if ('usr'+username).encode('ascii','ignore') in database: auth_info = json.loads( database[('usr'+username).encode('ascii','ignore')] ) if auth_info['password'] == password: session_store = sessions.get_store(request=self.request) session = session_store.get_session() session['key'] = auth_info['key'] session['username'] = username session_store.save_sessions(self.response) database.close() self.redirect('/jj70/abstracts') else: database.close() self.redirect('/jj70/login') else: database.close() self.redirect('/jj70/login')
def show_user_timeline(user, since_ids = None, hashtag_list = None, hashtag_db_name = None, tweet_db_name = None): ''' user - twitter.User object since_ids - twitter.Status.id (the timeline will consist of all the tweets after the tweet with id as since_ids) hashtag_list - A list of hashtags if you want to search for particular hashtags hashtag_db_name - Provide a string name, If you want to save the hashtags and the counts. It will be stored in a gdbm file. tweet_db_name - Provide a string name, If you want to save the tweets hashtags and the tweet id. It will be stored in a gdbm file. Returns the tweet id of the latest tweet. ''' if not user: return since_ids if not user.protected: try: timeline = api.GetUserTimeline(user.id, since_id = since_ids) except ValueError: print 'ValueError' else: return since_ids if not timeline: return since_ids hashtag_user_db = None if hashtag_db_name: # print hashtag_db_name hashtag_user_db = gdbm.open(hashtag_db_name+'_hashtag','c') if tweet_db_name: tweet_user_db = gdbm.open(tweet_db_name+'_tweets','c') since_ids = show_timeline(timeline, hashtag_db = hashtag_user_db, tweet_db = tweet_user_db, hashtag_list = hashtag_list) if hashtag_db_name: hashtag_user_db.close() if tweet_db_name: tweet_user_db.close() return since_ids
def scan_log(self, statedb_path, maildb_path, maintainer, email, logfile): self._state['maintainer'] = str(maintainer) self._state['email'] = email self._state['statedb'] = str(statedb_path) self._state['maildb'] = str(maildb_path) if not os.path.isfile(self._path['log']): raise RuntimeError("log file (" + self._path['log'] + ") does not exist") log = open(self._path['log'], "r") num = re.compile(r".*?([0-9.]+).*?") for line in log: line = line.rstrip() if line.startswith("Regression PASSED"): self._state['result'] = 'pass' elif line == 'status = pass # execution status': # from publish_summary self._state['result'] = 'pass' elif line == 'status = fail # execution status': # from publish_summary self._state['result'] = 'fail' elif line.startswith('time = '): # from publish_summary self._state['time:wall'] = re.match(num, line).group(1) elif line.startswith("Total wall clock time was"): self._state['time:wall'] = re.match(num, line).group(1) elif line.startswith("Total CPU time was"): self._state['time:cpu'] = re.match(num, line).group(1) elif line.startswith("[result:exit status]"): self._state['status'] = re.match(num, line).group(1) elif line.startswith("[result:exit condition]"): self._state['condition'] = re.match( r"\[result:exit condition\]\s+(.*)", line).group(1) log.close() for k, v in self._state.iteritems(): print str(k) + " -> " + str(v) divider = ";;" self._state['stamp'] = str( self._state['script']).split('.')[0] + ":" + self._state['time'] statedb = gdbm.open(self._state['statedb'], "c") statedb[self._state['stamp']] = (str(self._state['result']) if self._state.has_key('result') else "malfunction") + divider + \ (str(self._state['time:wall']) if self._state.has_key('time:wall') else "0") + divider + \ (str(self._state['time:cpu']) if self._state.has_key('time:cpu') else "0") statedb.close() if not self._state.has_key('result'): maildb = gdbm.open(self._state['maildb'], "c") maildb[self._state['stamp']] = "malfunction" + divider + str( self._state['script'] ) + divider + self._state['master'] + divider + self._state[ 'master-email'] + divider + logfile maildb.close() elif self._state['result'] != 'pass': maildb = gdbm.open(self._state['maildb'], "c") maildb[self._state['stamp']] = "fail" + divider + str( self._state['script']) + divider + str( maintainer) + divider + str(email) + divider + logfile maildb.close()
def __init__(self, filename, readonly): if readonly: # opens the database in readonly mode so that we don't get # conflicts between different processes accessing at the same # time. only the dbqueue can modify the database. self._dbm = gdbm.open(filename, 'ru') else: self._dbm = gdbm.open(filename, 'cf') feedlib.WordDatabase.__init__(self, self._dbm)
def kanwaout(self, out): try: unicode #python2 need .db ext dic = dbm.open(out+'.db', 'c') except: #python3 add automatically .db ext dic = dbm.open(out, 'c') for (k, v) in self.records.items(): dic[k] = compress(dumps(v)) dic.close()
def scan_log( self, statedb_path, maildb_path, maintainer, email, logfile ): self._state['maintainer'] = str(maintainer) self._state['email'] = email self._state['statedb'] = str(statedb_path) self._state['maildb'] = str(maildb_path) if not os.path.isfile(self._path['log']): raise RuntimeError("log file (" + self._path['log'] + ") does not exist") log = open( self._path['log'], "r" ) num = re.compile(r".*?([0-9.]+).*?") for line in log: line = line.rstrip( ) if line.startswith("Regression PASSED"): self._state['result'] = 'pass' elif line == 'OK': # from UNIT TEST self._state['result'] = 'pass' elif line.startswith("OK (SKIP="): #from UNIT TEST (special cases with skipped tests) self._state['result'] = 'pass' elif line == 'status = pass # execution status': # from publish_summary self._state['result'] = 'pass' elif line == 'status = fail # execution status': # from publish_summary self._state['result'] = 'fail' elif line.startswith('time = '): # from publish_summary self._state['time:wall'] = re.match(num,line).group(1) elif line.startswith("Total wall clock time was"): self._state['time:wall'] = re.match(num,line).group(1) elif line.startswith("Total CPU time was"): self._state['time:cpu'] = re.match(num,line).group(1) elif line.startswith("[result:exit status]"): self._state['status'] = re.match(num,line).group(1) elif line.startswith("[result:exit condition]"): self._state['condition'] = re.match(r"\[result:exit condition\]\s+(.*)",line).group(1) log.close( ) for k, v in self._state.iteritems( ): print str(k) + " -> " + str(v) divider = ";;" self._state['stamp'] = str(self._state['script']).split('.')[0] + ":" + self._state['time'] statedb = gdbm.open(self._state['statedb'],"c") statedb[self._state['stamp']] = (str(self._state['result']) if self._state.has_key('result') else "malfunction") + divider + \ (str(self._state['time:wall']) if self._state.has_key('time:wall') else "0") + divider + \ (str(self._state['time:cpu']) if self._state.has_key('time:cpu') else "0") statedb.close( ) if not self._state.has_key('result'): maildb = gdbm.open(self._state['maildb'],"c") maildb[self._state['stamp']] = "malfunction" + divider + str(self._state['script']) + divider + self._state['master'] + divider + self._state['master-email'] + divider + logfile maildb.close( ) elif self._state['result'] != 'pass': maildb = gdbm.open(self._state['maildb'],"c") maildb[self._state['stamp']] = "fail" + divider + str(self._state['script']) + divider + str(maintainer) + divider + str(email) + divider + logfile maildb.close( )
def opendb(ro=False): global jobs while jobs == None: try: if ro and exists(jobsfn): jobs = gdbm.open(jobsfn,'r') else: jobs = gdbm.open(jobsfn,'cs') except gdbm.error: time.sleep(1)
def test_error_conditions(self): # Try to open a non-existent database. unlink(filename) self.assertRaises(gdbm.error, gdbm.open, filename, 'r') # Try to access a closed database. self.g = gdbm.open(filename, 'c') self.g.close() self.assertRaises(gdbm.error, lambda: self.g['a']) # try pass an invalid open flag self.assertRaises(gdbm.error, lambda: gdbm.open(filename, 'rx').close())
def opendb(ro=False): global jobs while jobs == None: try: if ro and exists(jobsfn): jobs = gdbm.open(jobsfn, 'r') else: jobs = gdbm.open(jobsfn, 'cs') except gdbm.error: time.sleep(1)
def combine_databases(srcdb, destdb, cutoff=20, start=0): try: import gdbm except ImportError: print('combining databases requires the gdbm module. :(') print('adding tweets from %s to %s' % (srcdb, destdb)) db1 = gdbm.open(destdb, 'wf') db2 = gdbm.open(srcdb, 'w') k = db2.firstkey() temp_k = None seen = 0 # if not start: # start = 10**10 if start: seen = 0 while seen < start: k = db2.nextkey(k) sys.stdout.write('skipping: %i/%i \r' % (seen, start)) sys.stdout.flush() seen += 1 try: while k is not None: stats.tweets_seen() if (anagramfunctions.length_from_hash(k) < cutoff): k = db2.nextkey(k) continue stats.passed_filter() tweet = _tweet_from_dbm(db2[k]) if k in db1: tweet2 = _tweet_from_dbm(db1[k]) if anagramfunctions.test_anagram( tweet['tweet_text'], tweet2['tweet_text'] ): temp_k = db2.nextkey(k) del db2[k] hitmanager.new_hit(tweet, tweet2) else: pass else: db1[k] = _dbm_from_tweet(tweet) stats.update_console() k = db2.nextkey(k) if not k and temp_k: k = temp_k temp_k = None finally: db1.sync() db1.close() db2.close()
def run(self, yes): global force, verbose, HAVE_HDBM, HAVE_DBM if HAVE_GDBM: self.archive.cacheDb = gdbm.open(self.archive.cacheFile, 'nf') elif HAVE_DBM: self.archive.cacheDb = gdbm.open(self.archive.cacheFile, 'n') for indexFile in self.archive.list(): self.processIndexFile(indexFile) if HAVE_GDBM: self.archive.cacheDb.sync() if self.errors > 0 and not force: print sys.stderr.write("There where %d errors. Bailing out, no harm has been done to archive! Use '--force' to override.\n" % (self.errors)) else: for index0 in range(0, 256): level0Name = "%02x" % (index0) level0Path = os.path.join(self.archive.filesDir, level0Name) if verbose: print (" scanning " + level0Name) for index1 in range(0, 256): level1Name = "%02x" % (index1) level1Path = os.path.join(level0Path, level1Name) for filename in os.listdir(level1Path): self.filesInArchive += 1 remainder = filename[0:36] fullname = level0Name + level1Name + remainder if fullname not in self.archive.cacheDb: self.filesDeleted += 1 path = os.path.join(level1Path, filename) if verbose: print (" deleting " + fullname) if yes: os.remove(path) else: self.filesToKeep += 1 print print ("Files to keep: %12d" % self.filesToKeep) print ("Files deleted: %12d" % self.filesDeleted) print ("Files found in archive: %12d" % self.filesInArchive) if not yes: print print ("This was a dry run. Nothing has been deleted for real. Use '--yes' for a live run.") self.archive.cacheDb.close()
def combine_databases(srcdb, destdb, cutoff=20, start=0): try: import gdbm except ImportError: print('combining databases requires the gdbm module. :(') print('adding tweets from %s to %s' % (srcdb, destdb)) db1 = gdbm.open(destdb, 'wf') db2 = gdbm.open(srcdb, 'w') k = db2.firstkey() temp_k = None seen = 0 # if not start: # start = 10**10 if start: seen = 0 while seen < start: k = db2.nextkey(k) sys.stdout.write('skipping: %i/%i \r' % (seen, start)) sys.stdout.flush() seen += 1 try: while k is not None: stats.tweets_seen() if (anagramfunctions.length_from_hash(k) < cutoff): k = db2.nextkey(k) continue stats.passed_filter() tweet = _tweet_from_dbm(db2[k]) if k in db1: tweet2 = _tweet_from_dbm(db1[k]) if anagramfunctions.test_anagram(tweet['tweet_text'], tweet2['tweet_text']): temp_k = db2.nextkey(k) del db2[k] hitmanager.new_hit(tweet, tweet2) else: pass else: db1[k] = _dbm_from_tweet(tweet) stats.update_console() k = db2.nextkey(k) if not k and temp_k: k = temp_k temp_k = None finally: db1.sync() db1.close() db2.close()
def post(self): try: database = gdbm.open('database.gdbm','cf') except: time.sleep(1.0) database = gdbm.open('database.gdbm','cf') session_store = sessions.get_store(request=self.request) session = session_store.get_session() username = session.get('username') key = session.get('key') database['abstract'+key.encode('ascii','ignore')] = self.request.body database.close()
def reset(self): """ Resets the database - clears the ondisk stores and wipes cache """ self.id_idx_db.close() self.num_idx_db.close() self.blockdata_db.close() self.party_idxa_db.close() self.party_idxb_db.close() self.mc.flush_all() self.id_idx_db = gdbm.open('blockid_idx.db', 'nf') self.num_idx_db = gdbm.open('blocknum_idx.db', 'nf') self.blockdata_db = gdbm.open('blockdata.db', 'nf') self.party_idxa_db = gdbm.open('party_idx_a.db', 'nf') self.party_idxb_db = gdbm.open('party_idx_b.db', 'nf')
def test_flags(self): # Test the flag parameter open() by trying all supported flag modes. all = set(gdbm.open_flags) # Test standard flags (presumably "crwn"). modes = all - set('fsu') for mode in modes: self.g = gdbm.open(filename, mode) self.g.close() # Test additional flags (presumably "fsu"). flags = all - set('crwn') for mode in modes: for flag in flags: self.g = gdbm.open(filename, mode + flag) self.g.close()
def Open(self, indexfile=None): if not indexfile: indexfile = os.path.join(os.environ['PYPHY'], 'nr.dat.indexed') self.db = gdbm.open(indexfile) self.datafile = self.db['datafile'] self.fid = open(self.datafile)
def Create(self, infile, outfile): db = gdbm.open(outfile, 'n') fid = open(infile) db['datafile'] = os.path.abspath(infile) while 1: line = fid.readline() if not line or not len(line): break if line[:3] == 'ID ': id = string.split(line)[1] start = fid.tell() - len(line) elif line[:3] == 'AC ': acc = string.split(line)[1] if acc[-1] == ';': acc = acc[:-1] elif line[:2] == '//': stop = fid.tell() try: value = '%d %d' % (start, stop) db[id] = value db[acc] = value id, acc, start, stop = None, None, None, None except: print("AARRGGGG %d %d %s %s" % (start, stop, type(start), type(stop))) print("%s %s" % (id, acc)) db.close() fid.close()
def getBookmarkDict(filename): ''' Input: Absolute path of browser bookmark backup file Creates/Updates Bookamrk-Manager database Returns a dictionary object with bookmark url as key and (title, tag, add_date) tuple as value. ''' f = gdbm.open('bookmarkDB','c') bookmark_dict = fetchBookmarks(filename) if bookmark_dict: for key in bookmark_dict: f[key] = pickle.dumps(bookmark_dict[key]) if f.keys(): bookmark_dict = {} for key in f.keys(): bookmark_dict[key] = pickle.loads(f[key]) if not f.has_key('@author@'): name = 'Jay Rambhia' email = '*****@*****.**' add_date = time.time() f['@author@'] = pickle.dumps((name, email, add_date)) print 'bookmarks saved' f.close() return bookmark_dict
def get_repo_data(repo_ids): """ Find, open, and return the gdbm database file associated with each repo plus that repo's publish protocol :param repo_ids: list of repository IDs. :type repo_ids: list :return: dictionary where keys are repo IDs, and values are dicts that contain an open gdbm database under key "db", and a protocol under key "protocol". :rtype: dict """ ret = {} for distributor in RepoDistributorManager.find_by_repo_list(repo_ids): publish_protocol = _get_protocol_from_distributor(distributor) protocol_key, protocol_default_value = PROTOCOL_CONFIG_KEYS[ publish_protocol] repo_path = distributor['config'].get(protocol_key, protocol_default_value) repo_id = distributor['repo_id'] db_path = os.path.join(repo_path, repo_id, constants.REPO_DEPDATA_FILENAME) try: ret[repo_id] = { 'db': gdbm.open(db_path, 'r'), 'protocol': publish_protocol } except gdbm.error: _LOGGER.error( 'failed to find dependency database for repo %s. re-publish to fix.' % repo_id) return ret
def get_repo_data(repo_ids): """ Find, open, and return the gdbm database file associated with each repo plus that repo's publish protocol :param repo_ids: list of repository IDs. :type repo_ids: list :return: dictionary where keys are repo IDs, and values are dicts that contain an open gdbm database under key "db", and a protocol under key "protocol". :rtype: dict """ ret = {} for distributor in RepoDistributorManager.find_by_repo_list(repo_ids): publish_protocol = _get_protocol_from_distributor(distributor) protocol_key, protocol_default_value = PROTOCOL_CONFIG_KEYS[publish_protocol] repo_path = distributor['config'].get(protocol_key, protocol_default_value) repo_id = distributor['repo_id'] db_path = os.path.join(repo_path, repo_id, constants.REPO_DEPDATA_FILENAME) try: ret[repo_id] = {'db': gdbm.open(db_path, 'r'), 'protocol': publish_protocol} except gdbm.error: _LOGGER.error('failed to find dependency database for repo %s. re-publish to fix.' % repo_id) return ret
def main(): n = 42 f = gdbm.open('contacts', 'c') while n != 5: print '1. Add contact\n2. Search contact\n3. Show All contacts\n4. List according to char\n5. Quit' while True: try: n = int(raw_input('Enter: ')) break except ValueError: print 'Please choose a valid option' if n == 1: f = add_contact(f) elif n == 2: search_contact(f) elif n == 5: print 'Quit.\n' return elif n == 3: show_list(f) elif n == 4: c = raw_input('Enter the character.') show_alpha_list(f, c) else: print 'Please enter a valid option' return
def add_repodata(self, model): """ Given a model, add the "repodata" attribute to it (which includes raw XML used for publishing), and add the "files" and "changelog" attributes based on data obtained in the raw XML snippets. :param model: model instance to manipulate :type model: pulp_rpm.plugins.db.models.RpmBase """ db_key = self.generate_db_key(model.unit_key) for filename, metadata_key, process_func in ( (filelists.METADATA_FILE_NAME, 'files', filelists.process_package_element), (other.METADATA_FILE_NAME, 'changelog', other.process_package_element) ): try: db_file = gdbm.open(self.dbs[filename], 'r') raw_xml = db_file[db_key] finally: db_file.close() model.set_repodata(filename, raw_xml) element = ElementTree.fromstring(raw_xml) unit_key, items = process_func(element) setattr(model, metadata_key, items) primary_raw_xml = model.raw_xml.decode('utf-8', 'replace') primary_raw_xml = change_location_tag(primary_raw_xml, model.filename) model.set_repodata('primary', primary_raw_xml.encode('utf-8'))
def generate_dbs(self): """ For repo data files that contain data we need to access later for each unit in the repo, generate a local db file that gives us quick read access to each unit's data. :raises PulpCodedException: if there is some inconsistency in metadata """ package_count = {} for filename, tag, process_func in ( (filelists.METADATA_FILE_NAME, filelists.PACKAGE_TAG, filelists.process_package_element), (other.METADATA_FILE_NAME, other.PACKAGE_TAG, other.process_package_element), ): with contextlib.closing(self.get_metadata_file_handle(filename)) as xml_file_handle: generator = package_list_generator(xml_file_handle, tag) db_filename = os.path.join(self.dst_dir, '%s.db' % filename) # always a New file, and open with Fast writing mode. with contextlib.closing(gdbm.open(db_filename, 'nf')) as db_file_handle: for element in generator: utils.strip_ns(element) element.attrib['pkgid'] = models.RpmBase.PKGID_TEMPLATE raw_xml = utils.element_to_raw_xml(element) unit_key, _ = process_func(element) db_key = self.generate_db_key(unit_key) db_file_handle[db_key] = raw_xml db_file_handle.sync() package_count[filename] = len(db_file_handle) self.dbs[filename] = db_filename if package_count[filelists.METADATA_FILE_NAME] != package_count[other.METADATA_FILE_NAME]: reason = ('metadata is specified for different set of packages in filelists.xml' ' and in other.xml') raise PulpCodedException(error_code=error_codes.RPM1015, reason=reason) self.rpm_count = package_count[filelists.METADATA_FILE_NAME]
def __init__(self, filename): self.db = None if filename.endswith(".db"): try: self.db = gdbm.open(filename, "r") except gdbm.error, err: print >> sys.stderr, "Unable to open binary database %s: %s" % (filename, err)
def _generate_dependency_data(self, modules): """ generate the dependency metadata that is required to provide the API that the "puppet module" tool uses. Store the metadata in a gdbm database at the root of the repo. Generating and storing it at publish time means the API requests will always return results that are in-sync with the most recent publish and are not influenced by more recent changes to the repo or its contents. :type modules: list of pulp.plugins.model.AssociatedUnit """ filename = os.path.join(self._build_dir(), constants.REPO_DEPDATA_FILENAME) _LOG.debug('generating dependency metadata in file %s' % filename) # opens a new file for writing and overwrites any existing file db = gdbm.open(filename, 'n') try: for module in modules: version = module.unit_key['version'] deps = module.metadata.get('dependencies', []) path = os.path.join(self._repo_path, self._build_relative_path(module)) value = {'file' : path, 'version' : version, 'dependencies' : deps} name = module.unit_key['name'] author = module.unit_key['author'] key = '%s/%s' % (author, name) if db.has_key(key): module_list = json.loads(db[key]) else: module_list = [] module_list.append(value) db[key] = json.dumps(module_list) finally: db.close()
def check_uids(): if len(sys.argv) < 2: print_help() return (3) domainname = sys.argv[1] if domainname == "": return (2) map_file = "/var/yp/%s/passwd.byuid" % domainname try: fh = gdbm.open(map_file, 'r') if not fh: return (2) except gdbm.error as e: sys.stderr.write("Cannot open the %s NIS map file for reading: %s" % (map_file, e)) return (2) uid = fh.firstkey() while uid: uid = fh.nextkey(uid) if uid_is_dangerous(uid): return 0 fh.close() return 1
def __init__(self, filename = "/usr/lib64/avahi/service-types.db"): self.db = gdbm.open(filename, "r") l = locale.getlocale(locale.LC_MESSAGES) self.suffixes = () if not l[0] is None: if not l[1] is None: self.suffixes += (l[0] + "@" + l[1], ) self.suffixes += (l[0], ) i = l[0].find("_") if i >= 0: k = l[0][:i] if not l[1] is None: self.suffixes += (k + "@" + l[1], ) self.suffixes += (k, ) self.suffixes = tuple(map(lambda x: "["+x+"]", self.suffixes)) + ("", )
def __init__(self, fn, mode, max_age=None): self.max_age = max_age self.db = gdbm.open(fn, mode) self.reorganize_timer = None self.sync_timer = None self.start_reorganizing() self.start_syncing()
def __init__(self, filename="/usr/lib/x86_64-linux-gnu/avahi/service-types.db"): self.db = gdbm.open(filename, "r") l = locale.getlocale(locale.LC_MESSAGES) self.suffixes = () if not l[0] is None: if not l[1] is None: self.suffixes += (l[0] + "@" + l[1], ) self.suffixes += (l[0], ) i = l[0].find("_") if i >= 0: k = l[0][:i] if not l[1] is None: self.suffixes += (k + "@" + l[1], ) self.suffixes += (k, ) self.suffixes = tuple(map(lambda x: "[" + x + "]", self.suffixes)) + ("", )
def clean_database(databasePathname): """function to remove dead nodes from the hash db""" print '# loading database ' + databasePathname try: db = gdbm.open(databasePathname, 'w') except: print "# " + databasePathname + " could not be loaded" sys.exit(-1) # even though gdbm supports memory efficient iteration over # all keys, I want to order my traversal across similar # paths to leverage caching of directory files: allKeys=db.keys() print '# finished loaded keys from ' + databasePathname allKeys.sort() print '# finished sorting keys from ' + databasePathname print '# deleting dead nodes' count=0 for currKey in allKeys: try: os.stat(currKey) sys.stdout.write('.') except OSError: del db[currKey] sys.stdout.write('*') count=count+1 sys.stdout.flush() print "\n# reorganizing " + databasePathname db.reorganize() db.sync() db.close() print '# done cleaning ' + databasePathname + ', removed ' + str(count) + ' dead nodes!'
def open_gdbm(*args): """Open a gdbm database.""" import gdbm return gdbm.open(*args)
def generate_dbs(self): """ For repo data files that contain data we need to access later for each unit in the repo, generate a local db file that gives us quick read access to each unit's data. """ for filename, tag, process_func in ( (filelists.METADATA_FILE_NAME, filelists.PACKAGE_TAG, filelists.process_package_element), (other.METADATA_FILE_NAME, other.PACKAGE_TAG, other.process_package_element), ): xml_file_handle = self.get_metadata_file_handle(filename) try: generator = package_list_generator(xml_file_handle, tag) db_filename = os.path.join(self.dst_dir, '%s.db' % filename) # always a New file, and open with Fast writing mode. db_file_handle = gdbm.open(db_filename, 'nf') try: for element in generator: utils.strip_ns(element) raw_xml = utils.element_to_raw_xml(element) unit_key, _ = process_func(element) db_key = self.generate_db_key(unit_key) db_file_handle[db_key] = raw_xml db_file_handle.sync() finally: db_file_handle.close() finally: xml_file_handle.close() self.dbs[filename] = db_filename
def getBookmarkDict(filename): ''' Input: Absolute path of browser bookmark backup file Creates/Updates Bookamrk-Manager database Returns a dictionary object with bookmark url as key and (title, tag, add_date) tuple as value. ''' f = gdbm.open('bookmarkDB', 'c') bookmark_dict = fetchBookmarks(filename) if bookmark_dict: for key in bookmark_dict: if not f.has_key(key): f[key] = pickle.dumps(bookmark_dict[key]) if f.keys(): bookmark_dict = {} for key in f.keys(): bookmark_dict[key] = pickle.loads(f[key]) if not f.has_key('@author@'): name = 'Jay Rambhia' email = '*****@*****.**' add_date = time.time() f['@author@'] = pickle.dumps((name, email, add_date)) print 'bookmarks saved' f.close() return bookmark_dict
def add_repodata(self, model): """ Given a model, add the "repodata" attribute to it (which includes raw XML used for publishing), and add the "files" and "changelog" attributes based on data obtained in the raw XML snippets. :param model: model instance to manipulate :type model: pulp_rpm.plugins.db.models.RPM """ repodata = model.metadata.setdefault('repodata', {}) db_key = self.generate_db_key(model.unit_key) for filename, metadata_key, process_func in ( (filelists.METADATA_FILE_NAME, 'files', filelists.process_package_element), (other.METADATA_FILE_NAME, 'changelog', other.process_package_element)): try: db_file = gdbm.open(self.dbs[filename], 'r') raw_xml = db_file[db_key] finally: db_file.close() repodata[filename] = raw_xml element = ElementTree.fromstring(raw_xml) unit_key, items = process_func(element) model.metadata[metadata_key] = items raw_xml = model.raw_xml repodata['primary'] = change_location_tag(raw_xml, model.relative_path)
def Create(self, infile, outfile): db = gdbm.open(outfile, 'n') fid = open(infile) db['datafile'] = os.path.abspath(infile) while True: line = fid.readline() if not line or not len(line): break if line[:3] == 'ID ': id = string.split(line)[1] start = fid.tell() - len(line) elif line[:3] == 'AC ': acc = string.split(line)[1] if acc[-1] == ';': acc = acc[:-1] elif line[:2] == '//': stop = fid.tell() try: value = '%d %d' % (start, stop) db[id] = value db[acc] = value id, acc, start, stop = None, None, None, None except: print("AARRGGGG %d %d %s %s" % (start, stop, type(start), type(stop))) print("%s %s" % (id, acc)) db.close() fid.close()
def show_timeline(timeline, hashtag_db=None, tweet_db = None, hashtag_list=None): for i in range(len(timeline)-1,-1,-1): ids = timeline[i].id screen_name = '@'+timeline[i].user.screen_name user_name = timeline[i].user.name text = timeline[i].text tweet = screen_name+' ('+user_name+') '+': '+text print tweet res = get_hashtag(text) if hashtag_list: for j in range(len(hashtag_list)): if not hashtag_list[j].startswith('#'): hashtag_list[j]='#'+hashtag_list[j] if hashtag_list[j] in res: # print "opening",hashtag_list[j]+"_hashtag" py_db = gdbm.open(hashtag_list[j]+'_hashtag','c') py_db[str(timeline[i].id)] = repr(tweet) py_db.close() if res: # print hashtag_db if hashtag_db is not None: # print 'save_hashtag' hashtag_db = save_hashtag(res, hashtag_db) if tweet_db is not None: tweet_db = save_tweet(ids, tweet, tweet_db) return timeline[0].id