def __init__(self, directory ,**kwargs ):
""" Create a new controller object for our simplistic TxF implementation
recover :bool - False to force skipping journal recovery.
reserve : int - initial minimum size for log file.
"""
self.home = directory
self.track_state = kwargs.get('track_state',True)
self.last_opid = -1 #cleanup
try:
os.remove(tstfile)
except:
pass
#id_lock prevents new transactions being created
# and protect last_opid.
self.id_lock = threading.Lock()
#not_complete_lock protects the not_complete list, prevents
#transaction being submitted and is used for the
#outstanding condition variable. Always acquire not_complete_lock
#before id_lock
self.oldlogtx = -1
self.tx = None
if kwargs.pop('recover',True): self._recover()
self.frozen = False #FIXME: In the case that this is a new file
#we also should sync the directory fd (which means opening one).
self.logf = None
self.in_use_logs = []
self.logsync = threading.Semaphore(0)
self.loglocker = RWLock()
self.rotatelog(**kwargs)
class FileLoggerSimpleFS(object):
def __init__(self, directory ,**kwargs ):
""" Create a new controller object for our simplistic TxF implementation
recover :bool - False to force skipping journal recovery.
reserve : int - initial minimum size for log file.
"""
self.home = directory
self.track_state = kwargs.get('track_state',True)
self.last_opid = -1 #cleanup
try:
os.remove(tstfile)
except:
pass
#id_lock prevents new transactions being created
# and protect last_opid.
self.id_lock = threading.Lock()
#not_complete_lock protects the not_complete list, prevents
#transaction being submitted and is used for the
#outstanding condition variable. Always acquire not_complete_lock
#before id_lock
self.oldlogtx = -1
self.tx = None
if kwargs.pop('recover',True): self._recover()
self.frozen = False #FIXME: In the case that this is a new file
#we also should sync the directory fd (which means opening one).
self.logf = None
self.in_use_logs = []
self.logsync = threading.Semaphore(0)
self.loglocker = RWLock()
self.rotatelog(**kwargs)
def _findlogs(self,):
logs = glob.glob(os.path.join(*self._get_logname("*")))
modlogger.debug( "Logs:%s"%logs)
return logs
def _newname(self,):
names = self._findlogs()
avail = list(set(names) - set(self.in_use_logs))
if avail:
return os.path.join(self.home, avail[0])
else:
path, zeroth_log = self._get_logname(-1)
nr = max([x.split(".")[3] for x in chain([zeroth_log],names)])
nr = int(nr) +1
return os.path.join(*self._get_logname(nr))
def _get_logname(self,seq):
"""Takes integer decribing the sequence nr of the log file,
or * to generate a glob pattern for logfiles"""
return self.home,"..xaction.%s.log"%seq
def new_opid(self,):
"""Internal: Create a new unique transaction id (Thread safe)"""
with self.id_lock:
v = self.last_opid + 1
self.last_opid = v
return v
def _recover(self,):
"""Replay associated logfiles. This function blocks
until completion."""
modlogger.debug( "starting recovery")
with self.id_lock: #Prevent new ops being created.
logs = [ LogFile(x,readonly=True) for x in self._findlogs() ]
logiter = [ iter(x) for x in logs ]
ops = [ _getop(x) for x in logiter ]
opids = [ _getid(x) for x in ops ]
#order the log files by operation Id.
data = sorted(izip(logs,logiter,ops,opids),key =lambda x:x[3])
modlogger.debug( "SR:%s"%data)
#And now got through all log files in Id order
state = 'init'
unrecoverable = []
for log,it,op,opid in data:
for cur_op in chain([op],it):
#cur_op None indicated end of that logfile.
if cur_op is None: break
#We ignore any ops until we see a 'startTxn' marker, but we
# keep a record of there ids to ensure we see a later checkpoint.
# if we don't we can't replay partial Txn.
modlogger.debug( "R:",cur_op,state)
if state=='init':
#Record all operations we see before we see the first
#start tx marker.
if cur_op.optype == 'start_txn':
state='txcomplete'
elif cur_op.optype == 'abort_txn':
#If the partial transaction we found was aborted
# we don't need to worry about its operations.
unrcoverable = [ ]
elif cur_op.optype == 'Checkpoint':
unrecoverable = _remove_commited(unrecoverable,cur_op.opid)
else:
unrecoverable += [ op.opid]
#We are looking for a starttxn, marker to mark the operation
#as valid. The only other meaningful transaction in the
#journal in the state is a checkpoint making which ops have been
#detected as committed to the main store by the FS.
if state=='txcomplete':
if cur_op.optype == 'start_txn':
tx = cur_op.txn_id
self.txops = [ ]
state = 'txstarted'
continue
elif cur_op.optype == 'Checkpoint':
unrecoverable = _remove_commited(unrecoverable,cur_op.opid)
else: raise RecoveryError("Operation outside tx")
#In this state all operations are meaningful.
# we store all operations (except checkpoint) until we see
# a EndTxn op. At the end TxnOp we synchronously complete
# all operations.
if state =='txstarted':
if cur_op.optype == 'end_txn':
#The test below finds 'overlapped' tx, (or ones missing a commit record
#for some reason. This forces us not to accept this log file.
if cur_op.txn_id != tx: raise RecoveryError("Non matching Tx commit found")
else:
for top in self.txops:
top.do(sync = True)
state = 'txcomplete'
elif cur_op.optype == 'abort_txn':
state = 'txcomplete'
elif cur_op.optype == 'Checkpoint':
unrecoverable = _remove_commited(unrecoverable,cur_op.opid)
else:
self.txops += [ cur_op ]
#Log file has been processed successfully - remove it from the Fs.
#we could call close() here and reused the allocated space on the
#FS - but the logfile is readonly - and close() adds a terminator
#to mark the file as empty.
try:
log.unlink()
except OSError: pass
#If there are any partial txn's left we have failed to recover.
if unrecoverable: raise RecoveryError("Partial uncommitted txn found")
def rotatelog(self,**kwargs):
"""Create a new log file ready for use and place , maek it the
active file"""
newname = self._newname()
newlgf = LogFile(newname,**kwargs)
with self.id_lock:
self._rotatelog(newlgf,newname)
def _rotatelog(self,newlgf,newname):
"""Internal function. Should be called with id_lock held"""
modlogger.debug( "rl:%s"%newname)
if self.logf:
thread.start_new_thread(self._waitlog,(self.logf,self.logname))
self.logsync.acquire()
if newname: self.in_use_logs += [ newname ]
try:
self.logf, self.logname = newlgf , newname
except:
if newname:
self.in_use_logs.remove(newname)
raise
def _waitlog(self,logf,fname):
self.loglocker.acquire_read()
self.logsync.release()
logf.close()
self.in_use_logs.remove(fname)
self.loglocker.release()
def Add_File(self,txn,filename,newcontents):
"""Log new file content transaction."""
opid = self.new_opid()
fullname = os.path.join(self.home,filename)
#if not self.tx.dir_exists(os.path.dirname(fullname)):
# raise OSError(errno.ENOENT,"No directory: %r"%os.path.dirname(fullname))
xaction = ReplaceAll_Operation(fullname,newcontents,opid)
self._add_operation(txn,xaction)
def Delete_File(self,txn,filename):
"""Log a delete file transaction."""
opid = self.new_opid()
xaction = DeleteFile_Operation(os.path.join(self.home,filename),opid)
self._add_operation(txn,xaction)
def Create_Dir(self,txn,filename):
"""Log a delete file transaction."""
opid = self.new_opid()
xaction = CreateDir_Operation(os.path.join(self.home,filename),opid)
self._add_operation(txn,xaction)
def Delete_Dir(self,txn,filename):
"""Log a delete file transaction."""
opid = self.new_opid()
xaction = DeleteDir_Operation(os.path.join(self.home,filename),opid)
self._add_operation(txn,xaction)
def _add_operation(self,txn,xaction):
modlogger.debug( "ao")
if txn != self.tx.xid: raise RuntimeError("wrong txn %s!=%s"%(txn,self.tx.xid))
self.tx.add(xaction)
while not ( self.logf and self.logf.append(xaction)):
self.rotatelog(reserve = len(xaction))
def start_transaction(self,):
"""Start a new transaction to group operations.
Returns: an opaque obj for use with other mehtods"""
if self.tx is not None:
raise OverlappedTransaction(str(self.tx.xid))
modlogger.debug("start tx")
opid = self.new_opid()
xaction = StartTxOperation(opid,opid)
self.tx = Transaction(opid,self.home, track_state = self.track_state)
self._add_operation(opid,xaction)
return opid
def commit_transaction(self,xid):
"""Commit the tranasaction with id xid.
This forces all the operations which have been logged as
part of this transaction to persistent store.
"""
modlogger.debug( "end tx:%s"%xid)
if xid != self.tx.xid:
raise InvalidTransaction(xid)
opid = self.new_opid()
xaction = EndTxOperation(opid,xid)
self._add_operation(xid,xaction)
try:
self.tx.commit()
except:
self.tx = None
#There is an arguement to call abort here...
# but we don't because our (MysteryMachine) use
# of this is a bottom layer store, which means the
# schema should have already ensured the transactions
# are meaningful and a exceptio here is probably a
# due to local conditions (like diskspace).
#
# If we abort we would have to remove the commit record
# from the log, and then undo the partial tranasction.
# by not aborting, and re-raisig we make this the
# problem of a higher layer. Once the problem is resolved
# the journal can be replayied making ourselve consistent
# again.
#
# There is one way though of provoking non-recoverable behavour
# ad that is creating a file in a non-existant dir.
#self.abort_transaction(xid)
raise
self.tx = None
def abort_transaction(self,xid):
"""Aborts the transaction with id xid.
This forces all the operations which have been logged as
part of this transaction to persistent store.
"""
modlogger.debug( "abort:%s"%xid)
opid = self.new_opid()
xaction = AbortTxOperation(opid,xid)
self._add_operation(xid,xaction)
try:
self.tx.rollback()
finally:
self.tx = None
def close(self,):
"""Finish use of the File logged directory"""
self.freeze()
def freeze(self,):
"""Bring main store upto date and prevent any more
transactions while frozen"""
if self.frozen: return
self.id_lock.acquire()
#Set logfile to None. Put current logfile into wait for chkpt state.
self._rotatelog(None,"")
self.loglocker.acquire_write()
self.frozen = True
def unfreeze(self,):
"""Release any transactions from being blocked by the frozen state."""
if self.frozen and self.id_lock.locked():
self.id_lock.release()
self.loglocker.release()
self.frozen = False
thaw = unfreeze
def _del_(self,):
if not self.frozen:
#Depend on nothing outside this object
import logging
logging.getLogger("MysteryMachine.store.FileLogger").error("Detected cleanup of Fileloogger incorrectly closed")