def isDbUpToDate ( databaseFile ):
""" Check if index database is up to date """
upToDate = False
for dbFile in ( databaseFile + os.path.extsep + 'db', databaseFile ):
try:
if not os.path.exists(dbFile):
# Database doesn't exist, try next one
continue;
if not whichdb(dbFile):
# Not in a readable format
logger.debug ( 'Database not in a readable format: %s', dbFile );
break;
# From here, database exists and is readable
db = dbm.open(dbFile)
sourceFile = db["__source_path__"]
if not os.path.exists(sourceFile):
# Source file doesn't exist any more
break;
textFileSum = md5sum(sourceFile)
if textFileSum != db["__source_md5__"]:
logger.debug ( 'Source file checksum differs from the one used to build the database: %s', sourceFile );
db.close()
break;
if not db["__version__"] == Database.version:
logger.debug ( 'Database version "%s" doesn\'t match this version "%s"', db["__version__"], Database.version );
db.close()
break;
db.close()
# Everything is ok with the existing database
upToDate = True
break;
except Exception:
pass
return upToDate
def parse_confs(self,conf_dir):
try:
agents = {}
for conf in searchFiles(conf_dir, lambda _,ext: ext=='agent'):
conf = os.path.abspath(conf)
# Records this new agent placeholder
agents[conf] = Any( **{"sum":0, "current_sum":md5sum(conf), "handle":None} );
except Exception:
logger.error ( 'Exception parsing conf %s', sys.exc_info()[1] );
logger.debug ( "", exc_info=True );
return;
return agents
def refresh (self):
""" Rebuild the index from source file """
# The cache directory must exist
self.check_cache(confdir.cache)
# these might speed-up indexing
db_oids = []
open_flags = 'nfu'
while open_flags:
try:
# Issue #4: work on a temporary file to limit collisions
db = dbm.open(self.__dbFileTmp, open_flags)
db["__version__"] = Database.version
db["__source_path__"] = os.path.abspath(self.sourceFile)
db["__source_md5__"] = md5sum(self.sourceFile)
except Exception:
open_flags = open_flags[:-1]
if not open_flags:
raise
else:
break
text = open(self.sourceFile, 'rb')
logger.debug ( 'Building index %s for data file %s (open flags \"%s\")', self.__dbFileTmp, self.sourceFile, open_flags );
# Build the "direct" indexes to have direct access to values
nb_direct = nb_next = 0
lineNo = 0
while 1:
try:
oid = line = None
while not oid:
line = text.readline()
lineNo += 1
if not line: break
oid, tag, val = self.textParser.grammar.parse(line)
if not oid: break
except Exception:
db.close()
exc = sys.exc_info()[1]
try:
os.remove(self.__dbFileTmp)
except OSError:
pass
raise Exception('Data error at %s:%d: %s' % ( self.sourceFile, lineNo, exc ) )
try:
_oid = self.textParser.evaluateOid(oid)
except Exception:
db.close()
exc = sys.exc_info()[1]
try:
os.remove(self.__dbFileTmp)
except OSError:
pass
raise Exception( 'OID error at %s:%d: %s' % ( self.sourceFile, lineNo, exc ) )
try:
_tag = self.textParser.evaluateTag(tag)
except Exception:
logger.warn ( 'Validation error at line %s, tag %r: %s', lineNo, tag, sys.exc_info()[1] );
try:
_val = self.textParser.evaluateValue( oid, tag, val, dataValidation=True)
except Exception:
logger.warn ( 'Validation error at line %s, value %r: %s', lineNo, val, sys.exc_info()[1] );
# for lines serving subtrees, type is empty in tag field
db[oid] = '%s,%d,%s,%s' % (oid2str(_oid), tag[0] == ':', _tag, _val)
db_oids.append ( _oid );
nb_direct = nb_direct+1
# Build the "next" indexes to have direct access to next values
# First we need oids splitted into nodes. We cannot sort them by string
# comparison: "1"<"10"<"2" and we want 1<2<10
db_oids.sort()
for i in range(len(db_oids)-1):
oid = db_oids[i]
oid_txt = oid2str(oid)
# The easy one
key = "next."+oid_txt
db[key] = oid2str(db_oids[i+1])
nb_next = nb_next+1
# Now the parents: their next is current oid unless they already have one next
nodes = oid[:-1]
for n in range(len(nodes)):
key = "next." + oid2str(nodes[:n+1])
if not db.has_key(key):
db[key] = oid_txt
nb_next = nb_next+1
# The last one have no next
key = "next." + oid2str(db_oids[ len(db_oids)-1 ])
db[key] = ""
nb_next = nb_next+1
text.close()
db.close()
logger.debug ( 'Index ok: %d direct entries, %d next entries' % (nb_direct,nb_next) );
# Issue #4: Synchronizes access to the database
self.__lock.acquire(True);
try:
self.close()
if os.access(self.__dbFile, os.R_OK):
os.remove(self.__dbFile);
os.rename(self.__dbFileTmp, self.__dbFile);
self.__dbType = whichdb(self.__dbFile)
finally:
self.__lock.release();