def outReceived(self, data):
log.debugd("Received:", data)
for line in data.split('\n'):
if len(line) <= 0:
continue
(path, action, file) = line.split()
if file in self.blacklist:
continue
if action == "CREATE":
if file in self.deleted: # Usecase DELETE+CREATE
self.deleted.remove(file)
self.updated.append(file)
else:
self.added.append(file)
elif action == "DELETE":
if file in self.added: # Usecase CREATE+DELETE
self.added.remove(file)
else:
self.deleted.append(file)
if file in self.updated: # Usecase MODIFY+DELETE
self.updated.remove(file)
elif action == "MODIFY":
self.updated.append(file)
# Remove duplicate entries
self.added = list(set(self.added))
self.deleted = list(set(self.deleted))
self.updated = list(set(self.updated))
self.rescheduleCommit()
def outReceived(self, data):
log.debugd("Received:", data)
for line in data.split('\n'):
if len(line) <= 0:
continue
(path, action, file)=line.split()
if file in self.blacklist:
continue
if action == "CREATE":
if file in self.deleted: # Usecase DELETE+CREATE
self.deleted.remove(file)
self.updated.append(file)
else:
self.added.append(file)
elif action == "DELETE":
if file in self.added: # Usecase CREATE+DELETE
self.added.remove(file)
else:
self.deleted.append(file)
if file in self.updated: # Usecase MODIFY+DELETE
self.updated.remove(file)
elif action == "MODIFY":
self.updated.append(file)
# Remove duplicate entries
self.added=list(set(self.added))
self.deleted=list(set(self.deleted))
self.updated=list(set(self.updated))
self.rescheduleCommit()
def dispatchMessage(self, data, host):
dispatcher = {
"slavehb" : self.updateNodeStatus,
"masterhb" : self.updateMasterStatus,
"voterequest" : self.voteForNewMaster,
"voteresponse" : self.recordVote,
}
try:
msg=MessageHelper.get(data, host)
log.debugd("Received", msg)
dispatcher[msg.type()](msg)
except (MessageError, KeyError), e:
log.err("Bad message from %s : %s , %s" % (host,data,e))
def dispatchMessage(self, data, host):
dispatcher = {
"slavehb": self.updateNodeStatus,
"masterhb": self.updateMasterStatus,
"voterequest": self.voteForNewMaster,
"voteresponse": self.recordVote,
}
try:
msg = MessageHelper.get(data, host)
log.debugd("Received", msg)
dispatcher[msg.type()](msg)
except (MessageError, KeyError), e:
log.err("Bad message from %s : %s , %s" % (host, data, e))
def doCommit(self):
# Semi-global variables used by inner functions
added = list()
deleted = list()
updated = list()
def commit():
if len(added) > 0:
self.node.run(
"svn add " +
" ".join(map(lambda x: core.cfg['VMCONF_DIR'] + x, added)))
if len(deleted) > 0:
self.node.run("svn delete " + " ".join(
map(lambda x: core.cfg['VMCONF_DIR'] + x, deleted)))
self.node.run(
"svn --non-interactive commit -m 'svnwatcher autocommit' " +
core.cfg['VMCONF_DIR'])
def commitEnded(result):
if self.agent:
log.info("Cluster updated.")
else:
log.info("Local node updated.")
def commitFailed(reason):
log.err("SVN failed: %s" % reason.getErrorMessage())
def releaseLock(result):
self.commitRunning = False
if self.agent:
self.agent.releaseLock(self.LOCK)
def checkLock(result):
# If result is true, no commit running (lock successfully grabbed)
if result:
# Get a local copy for thread's work
# Use .extend insted of = due to scope restriction (vars are in the parent function)
added.extend(self.added)
deleted.extend(self.deleted)
updated.extend(self.updated)
self.added = list()
self.deleted = list()
self.updated = list()
log.info("Committing for " +
", ".join(set(added + deleted + updated)))
self.commitRunning = True
d = threads.deferToThread(commit)
d.addCallback(lambda _: self.doUpdate())
d.addCallbacks(commitEnded, commitFailed)
d.addCallback(releaseLock)
return d
else:
log.info("Commit already running: rescheduling.")
self.rescheduleCommit()
return defer.succeed(None)
# Get files versionned in svn
versionned = [
line.split()[-1].replace(core.cfg['VMCONF_DIR'], '')
for line in self.node.run("svn status --verbose " +
core.cfg['VMCONF_DIR']).readlines()
]
# Keep the intersection: do not try to delete already deleted files
self.deleted = list(set(self.deleted) & set(versionned))
# Don't commit if there is no updates
if len(self.added + self.deleted + self.updated) <= 0:
log.debugd("Trigger commit: nothing to commit.")
return defer.succeed(None)
log.debugd("Trigger commit: ADD%s DEL%s UP%s" %
(self.added, self.deleted, self.updated))
# Check for locks of a previous commit still running
if self.agent:
# Cluster mode
d = self.agent.grabLock(self.LOCK)
d.addCallback(checkLock)
else:
# Standalone mode
d = checkLock(not self.commitRunning)
d.addErrback(log.err)
return d
def doCommit(self):
# Semi-global variables used by inner functions
added=list()
deleted=list()
updated=list()
def commit():
if len(added) > 0:
self.node.run("svn add " + " ".join(map(lambda x: core.cfg['VMCONF_DIR']+x, added)))
if len(deleted) > 0:
self.node.run("svn delete " + " ".join(map(lambda x: core.cfg['VMCONF_DIR']+x, deleted)))
self.node.run("svn --non-interactive commit -m 'svnwatcher autocommit' "+core.cfg['VMCONF_DIR'])
def commitEnded(result):
if self.agent:
log.info("Cluster updated.")
else:
log.info("Local node updated.")
def commitFailed(reason):
log.err("SVN failed: %s" % reason.getErrorMessage())
def releaseLock(result):
self.commitRunning=False
if self.agent:
self.agent.releaseLock(self.LOCK)
def checkLock(result):
# If result is true, no commit running (lock successfully grabbed)
if result:
# Get a local copy for thread's work
# Use .extend insted of = due to scope restriction (vars are in the parent function)
added.extend(self.added)
deleted.extend(self.deleted)
updated.extend(self.updated)
self.added=list()
self.deleted=list()
self.updated=list()
log.info("Committing for "+", ".join(set(added+deleted+updated)))
self.commitRunning=True
d=threads.deferToThread(commit)
d.addCallback(lambda _: self.doUpdate())
d.addCallbacks(commitEnded, commitFailed)
d.addCallback(releaseLock)
return d
else:
log.info("Commit already running: rescheduling.")
self.rescheduleCommit()
return defer.succeed(None)
# Get files versionned in svn
versionned=[
line.split()[-1].replace(core.cfg['VMCONF_DIR'],'')
for line in self.node.run("svn status --verbose "+core.cfg['VMCONF_DIR']).readlines()
]
# Keep the intersection: do not try to delete already deleted files
self.deleted=list(set(self.deleted) & set(versionned))
# Don't commit if there is no updates
if len(self.added+self.deleted+self.updated) <= 0:
log.debugd("Trigger commit: nothing to commit.")
return defer.succeed(None)
log.debugd("Trigger commit: ADD%s DEL%s UP%s" % (self.added, self.deleted, self.updated))
# Check for locks of a previous commit still running
if self.agent:
# Cluster mode
d=self.agent.grabLock(self.LOCK)
d.addCallback(checkLock)
else:
# Standalone mode
d=checkLock(not self.commitRunning)
d.addErrback(log.err)
return d
# Get diskhearbeat timestamps
try:
tsDisk=self.disk.get_all_ts()
except Exception, e:
log.err("Diskheartbeat read failed: %s." % (e))
raise
# Compare node lists from net and disk hearbeat
# Use Set's symmetric_difference
if len(Set(tsDisk.keys()) ^ Set(self.status.keys())) != 0:
log.err("Node list is inconsistent between net and disk heartbeats !")
raise Exception("Inconsistent internal data")
# Check disk heartbeat
log.debugd("Diskhearbeat status:", tsDisk)
diskFailed=Set()
for name, timestamp in tsDisk.items():
if timestamp == 0:
# Do nothing if first heartbeat has not been received yet
continue
# Timestamp from diskheartbeat is taken from each node, not relative to the master's time
# so we compute the absolute delta, with the time drift between nodes
if abs(int(time.time()) - timestamp - self.status[name]['offset']) >= MasterService.TM_SLAVE:
log.warn("Disk heartbeat lost for %s." % (name))
diskFailed.add(name)
# If we loose all heartbeats including master, there is a problem on localhost
if len(diskFailed) >= len(self.status) or len(netFailed) >= len(self.status):
log.err("Lost all heartbeats including me ! Maybe a clock screwed up ?")
# Get diskhearbeat timestamps
try:
tsDisk = self.disk.get_all_ts()
except Exception, e:
log.err("Diskheartbeat read failed: %s." % (e))
raise
# Compare node lists from net and disk hearbeat
# Use Set's symmetric_difference
if len(Set(tsDisk.keys()) ^ Set(self.status.keys())) != 0:
log.err(
"Node list is inconsistent between net and disk heartbeats !")
raise Exception("Inconsistent internal data")
# Check disk heartbeat
log.debugd("Diskhearbeat status:", tsDisk)
diskFailed = Set()
for name, timestamp in tsDisk.items():
if timestamp == 0:
# Do nothing if first heartbeat has not been received yet
continue
# Timestamp from diskheartbeat is taken from each node, not relative to the master's time
# so we compute the absolute delta, with the time drift between nodes
if abs(int(time.time()) - timestamp -
self.status[name]['offset']) >= MasterService.TM_SLAVE:
log.warn("Disk heartbeat lost for %s." % (name))
diskFailed.add(name)
# If we loose all heartbeats including master, there is a problem on localhost
if len(diskFailed) >= len(self.status) or len(netFailed) >= len(
