def __init__(self, history_id, object):
# The _versions mapping maps version ids to version objects. All
# of the actual version data is looked up there. The _labels
# mapping maps labels to specific version ids. The _branches map
# manages BranchInfo objects that maintain branch information.
self._eventLog = EventLog()
self._versions = OOBTree()
self._branches = OOBTree()
self._labels = OOBTree()
self.createBranch('mainline', None)
self.id = history_id
def __init__(self, history_id, object):
# The _versions mapping maps version ids to version objects. All
# of the actual version data is looked up there. The _labels
# mapping maps labels to specific version ids. The _branches map
# manages BranchInfo objects that maintain branch information.
self._eventLog = EventLog()
self._versions = OOBTree()
self._branches = OOBTree()
self._labels = OOBTree()
self.createBranch('mainline', None)
self.id = history_id
class VersionHistory(Implicit, Persistent):
"""A version history maintains the information about the changes
to a particular version-controlled resource over time."""
def __init__(self, history_id, object):
# The _versions mapping maps version ids to version objects. All
# of the actual version data is looked up there. The _labels
# mapping maps labels to specific version ids. The _branches map
# manages BranchInfo objects that maintain branch information.
self._eventLog = EventLog()
self._versions = OOBTree()
self._branches = OOBTree()
self._labels = OOBTree()
self.createBranch('mainline', None)
self.id = history_id
security = ClassSecurityInfo()
security.declarePublic('getId')
def getId(self):
return self.id
security.declarePrivate('addLogEntry')
def addLogEntry(self, version_id, action, path=None, message=''):
"""Add a new log entry associated with this version history."""
entry = LogEntry(version_id, action, path, message)
self._eventLog.addEntry(entry)
security.declarePrivate('getLogEntries')
def getLogEntries(self):
"""Return a sequence of the log entries for this version history."""
return self._eventLog.getEntries()
security.declarePrivate('getLabels')
def getLabels(self):
return self._labels.keys()
security.declarePrivate('labelVersion')
def labelVersion(self, version_id, label, force=0):
"""Associate a particular version in a version history with the
given label, removing any existing association with that label
if force is true, or raising an error if force is false and
an association with the given label already exists."""
current = self._labels.get(label)
if current is not None:
if current == version_id:
return
if not force:
raise VersionControlError(
'The label %s is already associated with a version.' % (
label
))
del self._labels[label]
self._labels[label] = version_id
security.declarePrivate('createBranch')
def createBranch(self, branch_id, version_id):
"""Create a new branch associated with the given branch_id. The
new branch is rooted at the version named by version_id."""
if self._branches.has_key(branch_id):
raise VersionControlError(
'Activity already exists: %s' % branch_id
)
branch = BranchInfo(branch_id, version_id)
self._branches[branch_id] = branch
return branch
security.declarePrivate('createVersion')
def createVersion(self, object, branch_id):
"""Create a new version in the line of descent named by the given
branch_id, returning the newly created version object."""
branch = self._branches.get(branch_id)
if branch is None:
branch = self.createBranch(branch_id, None)
if branch.name != 'mainline':
version_id = '%s.%d' % (branch.name, len(branch) + 1)
else:
version_id = '%d' % (len(branch) + 1)
version = ZopeVersion(version_id, object)
# Update the predecessor, successor and branch relationships.
# This is something of a hedge against the future. Versions will
# always know enough to reconstruct their lineage without the help
# of optimized data structures, which will make it easier to change
# internals in the future if we need to.
latest = branch.latest()
if latest is not None:
last = self._versions[latest]
last.next = last.next + (version_id,)
version.prev = latest
# If the branch is not the mainline, store the branch name in the
# version. Versions have 'mainline' as the default class attribute
# which is the common case and saves a minor bit of storage space.
if branch.name != 'mainline':
version.branch = branch.name
branch.append(version)
self._versions[version_id] = version
# Call saveState() only after version has been linked into the
# database, ensuring it goes into the correct database.
version.saveState(object)
return version.__of__(self)
security.declarePrivate('hasVersionId')
def hasVersionId(self, version_id):
"""Return true if history contains a version with the given id."""
return self._versions.has_key(version_id)
security.declarePrivate('isLatestVersion')
def isLatestVersion(self, version_id, branch_id):
"""Return true if version id is the latest in its branch."""
branch = self._branches[branch_id]
return version_id == branch.latest()
security.declarePrivate('getLatestVersion')
def getLatestVersion(self, branch_id):
"""Return the latest version object within the given branch, or
None if the branch contains no versions."""
branch = self._branches[branch_id]
version = self._versions[branch.latest()]
return version.__of__(self)
security.declarePrivate('findBranchId')
def findBranchId(self, version_id):
"""Given a version id, return the id of the branch of the version.
Note that we cheat, since we can find this out from the id."""
parts = version_id.split('.')
if len(parts) > 1:
return parts[-2]
return 'mainline'
security.declarePrivate('getVersionById')
def getVersionById(self, version_id):
"""Return the version object named by the given version id, or
raise a VersionControlError if the version is not found."""
version = self._versions.get(version_id)
if version is None:
raise VersionControlError(
'Unknown version id: %s' % version_id
)
return version.__of__(self)
security.declarePrivate('getVersionByLabel')
def getVersionByLabel(self, label):
"""Return the version associated with the given label, or None
if no version matches the given label."""
version_id = self._labels.get(label)
version = self._versions.get(version_id)
if version is None:
return None
return version.__of__(self)
security.declarePrivate('getVersionByDate')
def getVersionByDate(self, branch_id, timestamp):
"""Return the last version committed in the given branch on or
before the given time value. The timestamp should be a float
(time.time format) value in UTC."""
branch = self._branches[branch_id]
tvalue = int(timestamp / 60.0)
while 1:
# Try to find a version with a commit date <= the given time
# using the timestamp index in the branch information.
if branch.m_order:
try:
match = branch.m_date.maxKey(tvalue)
match = branch.m_order[branch.m_date[match]]
return self._versions[match].__of__(self)
except ValueError:
pass
# If we've run out of lineage without finding a version with
# a commit date <= the given timestamp, we return None. It is
# up to the caller to decide what to do in this situation.
if branch.root is None:
return None
# If the branch has a root (a version in another branch), then
# we check the root and do it again with the ancestor branch.
rootver = self._versions[branch.root]
if int(rootver.date_created / 60.0) < tvalue:
return rootver.__of__(self)
branch = self._branches[rootver.branch]
security.declarePrivate('getVersionIds')
def getVersionIds(self, branch_id=None):
"""Return a sequence of version ids for the versions in this
version history. If a branch_id is given, only version ids
from that branch will be returned. Note that the sequence
of ids returned does not include the id of the branch root."""
if branch_id is not None:
return self._branches[branch_id].versionIds()
return self._versions.keys()
class VersionHistory(Implicit, Persistent):
"""A version history maintains the information about the changes
to a particular version-controlled resource over time."""
def __init__(self, history_id, object):
# The _versions mapping maps version ids to version objects. All
# of the actual version data is looked up there. The _labels
# mapping maps labels to specific version ids. The _branches map
# manages BranchInfo objects that maintain branch information.
self._eventLog = EventLog()
self._versions = OOBTree()
self._branches = OOBTree()
self._labels = OOBTree()
self.createBranch('mainline', None)
self.id = history_id
security = ClassSecurityInfo()
security.declarePublic('getId')
def getId(self):
return self.id
security.declarePrivate('addLogEntry')
def addLogEntry(self, version_id, action, path=None, message=''):
"""Add a new log entry associated with this version history."""
entry = LogEntry(version_id, action, path, message)
self._eventLog.addEntry(entry)
security.declarePrivate('getLogEntries')
def getLogEntries(self):
"""Return a sequence of the log entries for this version history."""
return self._eventLog.getEntries()
security.declarePrivate('getLabels')
def getLabels(self):
return self._labels.keys()
security.declarePrivate('labelVersion')
def labelVersion(self, version_id, label, force=0):
"""Associate a particular version in a version history with the
given label, removing any existing association with that label
if force is true, or raising an error if force is false and
an association with the given label already exists."""
current = self._labels.get(label)
if current is not None:
if current == version_id:
return
if not force:
raise VersionControlError(
'The label %s is already associated with a version.' % (
label
))
del self._labels[label]
self._labels[label] = version_id
security.declarePrivate('createBranch')
def createBranch(self, branch_id, version_id):
"""Create a new branch associated with the given branch_id. The
new branch is rooted at the version named by version_id."""
if self._branches.has_key(branch_id):
raise VersionControlError(
'Activity already exists: %s' % branch_id
)
branch = BranchInfo(branch_id, version_id)
self._branches[branch_id] = branch
return branch
security.declarePrivate('createVersion')
def createVersion(self, object, branch_id):
"""Create a new version in the line of descent named by the given
branch_id, returning the newly created version object."""
branch = self._branches.get(branch_id)
if branch is None:
branch = self.createBranch(branch_id, None)
if branch.name != 'mainline':
version_id = '%s.%d' % (branch.name, len(branch) + 1)
else:
version_id = '%d' % (len(branch) + 1)
version = ZopeVersion(version_id, object)
# Update the predecessor, successor and branch relationships.
# This is something of a hedge against the future. Versions will
# always know enough to reconstruct their lineage without the help
# of optimized data structures, which will make it easier to change
# internals in the future if we need to.
latest = branch.latest()
if latest is not None:
last = self._versions[latest]
last.next = last.next + (version_id,)
version.prev = latest
# If the branch is not the mainline, store the branch name in the
# version. Versions have 'mainline' as the default class attribute
# which is the common case and saves a minor bit of storage space.
if branch.name != 'mainline':
version.branch = branch.name
branch.append(version)
self._versions[version_id] = version
# Call saveState() only after version has been linked into the
# database, ensuring it goes into the correct database.
version.saveState(object)
return version.__of__(self)
security.declarePrivate('hasVersionId')
def hasVersionId(self, version_id):
"""Return true if history contains a version with the given id."""
return self._versions.has_key(version_id)
security.declarePrivate('isLatestVersion')
def isLatestVersion(self, version_id, branch_id):
"""Return true if version id is the latest in its branch."""
branch = self._branches[branch_id]
return version_id == branch.latest()
security.declarePrivate('getLatestVersion')
def getLatestVersion(self, branch_id):
"""Return the latest version object within the given branch, or
None if the branch contains no versions."""
branch = self._branches[branch_id]
version = self._versions[branch.latest()]
return version.__of__(self)
security.declarePrivate('findBranchId')
def findBranchId(self, version_id):
"""Given a version id, return the id of the branch of the version.
Note that we cheat, since we can find this out from the id."""
parts = version_id.split('.')
if len(parts) > 1:
return parts[-2]
return 'mainline'
security.declarePrivate('getVersionById')
def getVersionById(self, version_id):
"""Return the version object named by the given version id, or
raise a VersionControlError if the version is not found."""
version = self._versions.get(version_id)
if version is None:
raise VersionControlError(
'Unknown version id: %s' % version_id
)
return version.__of__(self)
security.declarePrivate('getVersionByLabel')
def getVersionByLabel(self, label):
"""Return the version associated with the given label, or None
if no version matches the given label."""
version_id = self._labels.get(label)
version = self._versions.get(version_id)
if version is None:
return None
return version.__of__(self)
security.declarePrivate('getVersionByDate')
def getVersionByDate(self, branch_id, timestamp):
"""Return the last version committed in the given branch on or
before the given time value. The timestamp should be a float
(time.time format) value in UTC."""
branch = self._branches[branch_id]
tvalue = int(timestamp / 60.0)
while 1:
# Try to find a version with a commit date <= the given time
# using the timestamp index in the branch information.
if branch.m_order:
try:
match = branch.m_date.maxKey(tvalue)
match = branch.m_order[branch.m_date[match]]
return self._versions[match].__of__(self)
except ValueError:
pass
# If we've run out of lineage without finding a version with
# a commit date <= the given timestamp, we return None. It is
# up to the caller to decide what to do in this situation.
if branch.root is None:
return None
# If the branch has a root (a version in another branch), then
# we check the root and do it again with the ancestor branch.
rootver = self._versions[branch.root]
if int(rootver.date_created / 60.0) < tvalue:
return rootver.__of__(self)
branch = self._branches[rootver.branch]
security.declarePrivate('getVersionIds')
def getVersionIds(self, branch_id=None):
"""Return a sequence of version ids for the versions in this
version history. If a branch_id is given, only version ids
from that branch will be returned. Note that the sequence
of ids returned does not include the id of the branch root."""
if branch_id is not None:
return self._branches[branch_id].versionIds()
return self._versions.keys()
import logging
from DirectoryHandler import DirectoryHandler
from EventLog import EventLog
logging.basicConfig(
level=logging.DEBUG,
format=
'%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
datefmt='%a, %d %b %Y %H:%M:%S',
filename=r'./main.log',
filemode='w')
# define a stream that will show log level > Warning on screen also
console = logging.StreamHandler()
console.setLevel(logging.WARNING)
formatter = logging.Formatter('%(levelname)-8s %(message)s')
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
def main():
pass
if __name__ == '__main__':
input_directory = DirectoryHandler(r"./data/")
for x in input_directory.Target_File_Path:
EventLog(x)
def __init__(self, filename):
'''
Constructor.
'''
EventLog.__init__(self, filename)
self.__recordingToggle = itertools.cycle((False,True))
def main(argv):
if len(argv) != 3:
print 'Usage: ExtractUtterances <path_to_WAVE_Filename> <path_to_logfile>'
return
# get user's utterance segmentation
# wavFilename = 'pyAudioRecorder.2011-07-07_08-58-50-651998.wav'
# logFilename = '../experimentData/2011-07-07_08-58-47-503245/deepsemasr.devel.2011-07-07_08-58-47-503439.log.txt'
wavFilename = argv[1]
logFilename = argv[2]
if not os.path.exists(wavFilename):
print "Can't find audio file:",wavFilename
return
if not os.path.exists(logFilename):
print "Can't find log file:",logFilename
return
logDirname = os.path.split(logFilename)[0]
log = EventLog.parse(logFilename)
def filterAudioEvents(dt,ts,mt,msg):
if mt == 'AUDIO_RECORDING': return True
def filterStartRecording(dt,ts,mt,msg):
if mt == 'SCRIPT_STATE' and msg == 'started audio recording': return True
audioEventLog = EventLog.filter(log, filterAudioEvents)
startRecordingEvent = EventLog.filter(log, filterStartRecording)[0]
startRecordingTime, startRecordingTimestamp = startRecordingEvent[:2]
# load audio data
wf = wave.open(wavFilename, 'r')
if not(wf.getcompname() == 'NONE' or wf.getcompname() == 'not compressed'):
raise Exception('unsupported WAV audio compression') # need to use another wav-reading module?
sampFreqHz = wf.getframerate()
wavTime = startRecordingTime
uttIdx = 0
# cut at timestamps into "utt.wavTimestamp.uttIndex.wav"
outDirname = os.path.join(logDirname,'uttAudio.%s' % (startRecordingTimestamp))
os.mkdir(outDirname)
i = 0
while i < (len(audioEventLog) - 1):
if audioEventLog[i][3] != 'False':
print 'Warning: expected audio start event but not found'
i += 1
continue
j = i+1
while j < len(audioEventLog):
if audioEventLog[j][3] == 'True':
endFound = True
break
print 'Warning: expected audio stop event but not found'
j += 1
if not endFound:
print 'Error: start audio event with no corresponding stop audio event'
break
startTime = audioEventLog[i][0]
stopTime = audioEventLog[j][0]
i = j + 1
# skip ahead in wav file to start
timeToStart = startTime - wavTime
samplesToStart = int(math.floor(timeToStart.total_seconds() * sampFreqHz))
wf.readframes(samplesToStart)
wavTime += datetime.timedelta(seconds=(samplesToStart / sampFreqHz))
# read in/write out activated audio
audioDuration = stopTime - startTime
audioDurationSamples = int(math.ceil(audioDuration.total_seconds() * sampFreqHz))
wavOutFilename = os.path.join(outDirname,'utt.%s.%d.wav' % (startRecordingTimestamp, uttIdx))
print 'writing the %d seconds of utterance %d as: %s' % (audioDuration.total_seconds(), uttIdx, wavOutFilename)
wfout = wave.open(wavOutFilename, 'w')
wfout.setparams(wf.getparams())
wfout.writeframes(wf.readframes(audioDurationSamples))
wfout.close()
wavTime += datetime.timedelta(seconds=(audioDurationSamples / sampFreqHz))
uttIdx += 1
wf.close()
import datetime
producer = KafkaProducer(bootstrap_servers='kafka-7ab3a9e-justinraj1984-b417.aivencloud.com:14211',
security_protocol='SSL',
ssl_cafile='ca.pem',
ssl_certfile='access_cert.pem',
ssl_keyfile='key.pem')
try:
# Write My example Kafka message to exercise_topic
sent_msg = producer.send("exercise_topic", bytes(
"My example Kafka message", 'utf8')).get()
producer.flush()
print(sent_msg)
# Write a EventLog object to Kafka event_topic
newEvent = EventLog.EventLog(EventLog(), clientProcess="PID-"+str(os.getpid()),
inputLogMessage="Generated event message for process :"+str(os.getpid()), logDate=datetime.datetime.now())
print(""+str(newEvent.clientProcess) + ";" +
str(newEvent.inputLogMessage) + ";"+str(newEvent.logDate))
message = str(newEvent.clientProcess)+";"+str(
newEvent.inputLogMessage)+";"+str(newEvent.logDate)
sent_msg = producer.send(
"event_topic", value=bytes(message, "utf-8")).get()
producer.flush()
print(sent_msg)
except (Exception) as error:
print("Error: " + str(error.__str__))
print("Error: " + str(error))
