def increment(self, start, dur, segidx, selectCpsseg, logtext, corpusname):
cps_voiceid = selectCpsseg.voiceID
powers = selectCpsseg.desc['power']
cpsfilename = selectCpsseg.filename
self.p.log(logtext)
self.cnt['segidx'][segidx] += 1
self.cnt['onset'][start] += 1
self.cnt['selectionCount'] += 1
self.cnt['cpsnames'].append(corpusname)
self.histogram['select'].append(start)
self.choiceCnt += 1
for f in range(dur):
try:
if powers[f] >= self.overlap_inc_amp:
self.cnt['overlap'][start + f] += 1
self.cnt['cpsvc_overlap'][cps_voiceid][start + f] += 1
for str in self.cnt['corpusOverlapByString']:
if util.matchString(cpsfilename, str):
self.cnt['corpusOverlapByString'][str][0][start +
f] += 1
except IndexError:
break # cps handle data not long enough
def evaluate_midipitches(segmentobjlist, config, notfound=-1):
if config in ['filename', 'composite', 'centroid-seg', 'f0-seg']:
output_pitch_list = [MIDIPitchByFileName(obj.printName, config, obj, notfound=-1) for obj in segmentobjlist]
# float or int
elif type(config) in [float, int]:
output_pitch_list = [config for obj in segmentobjlist]
elif type(config) != dict:
util.error("SF SEGMENTS", 'midiPitchMethod must either be a string, a number, or a dictionary.')
elif 'type' not in config:
util.error("SF SEGMENTS", "a midiPitchMethod dictionary needs the key 'type'.")
# remap
elif config['type'] == 'remap': # {'type': 'remap', 'method': 'centroid-seg', 'high': 80, 'low': 76}
assert 'low' in config and 'high' in config and 'method' in config
pitchlist = np.array([MIDIPitchByFileName(obj.printName, config['method'], obj, notfound=-1) for obj in segmentobjlist])
minpitch, maxpitch = min(pitchlist), max(pitchlist)
output_pitch_list = (((pitchlist-minpitch)/(maxpitch-minpitch))*(config['high']-config['low']))+config['low']
# clip
elif config['type'] == 'clip':
assert 'low' in config or 'high' in config and 'method' in config
pitchlist = np.array([MIDIPitchByFileName(obj.printName, config['method'], obj, notfound=-1) for obj in segmentobjlist])
if not 'low' in config: config['low'] = None
if not 'high' in config: config['high'] = None
output_pitch_list = np.clip(pitchlist, config['low'], config['high'])
# filename string match
elif config['type'] == 'file_match':
output_pitch_list = [MIDIPitchByFileName(obj.printName, 'composite', obj, notfound=-1) for obj in segmentobjlist]
for k in config:
if k == 'type': continue
for cidx, c in enumerate(segmentobjlist):
if util.matchString(c.printName, k, caseSensative=True): output_pitch_list[cidx] = config[k]
else:
util.error("SF SEGMENTS", 'Ya done goofed son.')
# assignment..
for o, p in zip(segmentobjlist, output_pitch_list): o.midipitch = p
def pitchoverride(cobjlist, config):
pitchlist = [c.desc['MIDIPitch-seg'].get(None, None) for c in cobjlist]
minpitch, maxpitch = min(pitchlist), max(pitchlist)
output_dict = {}
for c, standardpitch in zip(cobjlist, pitchlist):
if config == None:
output_dict[c] = standardpitch
elif type(config) in [float, int]: # pitchoverride=60
output_dict[c] = config
elif type(config) != dict:
util.error("INSTRUMENTS", 'pitchoverride must either be None, a number, or a dictionary.')
# if passing this point, we're using the dict format
elif 'type' in config and config['type'] == 'remap':
assert 'low' in config and 'high' in config
standard_zerotoone = (standardpitch-minpitch)/(maxpitch-minpitch)
output_dict[c] = (standard_zerotoone*(config['high']-config['low']))+config['low']
# clip
elif 'type' in config and config['type'] == 'clip':
assert 'low' in config or 'high' in config
if 'low' in config and standardpitch < config['low']: output_dict[c] = config['low']
elif 'high' in config and standardpitch > config['high']: output_dict[c] = config['high']
else: output_dict[c] = standardpitch
# filename string match
elif 'type' in config and config['type'] == 'file_match':
for k in config:
if k == 'type': continue
if util.matchString(c.printName, k, caseSensative=True):
output_dict[c] = config[k]
# not found
output_dict[c] = standardpitch
else:
util.error("INSTRUMENTS", 'Ya done goofed son.')
return output_dict
def evaluateValidSamples(self, timeInFrames, timeInSec, tgtSegIdx,
rotateVoices, voicePattern,
voiceToCorpusIdMapping, clusterMappingDict,
tgtclusterId, superimp, instruments):
# get which voices are valid at this selection time
if rotateVoices and self.data['lastVoice'] != None:
validVoices = [
(self.data['lastVoice'] + 1) % self.data['numberVoices']
]
elif voicePattern not in [
None, []
]: # not case sensative!, does a search, so matches partial strings
validVoices = []
for nidx, name in enumerate(self.data['vcToCorpusName']):
if util.matchString(
name,
voicePattern[superimp.cnt['selectionCount'] %
len(voicePattern)],
caseSensative=False):
validVoices.append(nidx)
elif voiceToCorpusIdMapping not in [None, [], {}]:
realidx = tgtSegIdx % len(voiceToCorpusIdMapping)
if type(voiceToCorpusIdMapping[realidx]) in [list, tuple]:
validVoices = voiceToCorpusIdMapping[realidx]
else:
validVoices = [voiceToCorpusIdMapping[realidx]]
else:
validVoices = list(range(self.data['numberVoices']))
########################################################
## remove any voices that are outside temporal limits ##
########################################################
voicesToRemove = []
# voice frequency restriction per frame...
for vc in validVoices:
if vc in self.voiceRestrictPerFrame:
srt_look = int(
max(0, timeInFrames - self.voiceRestrictPerFrame[vc]))
end_look = int(
min(timeInFrames + self.voiceRestrictPerFrame[vc],
len(superimp.cnt['cpsvc_overlap'][vc])))
if np.sum(superimp.cnt['cpsvc_overlap'][vc]
[srt_look:end_look]) != 0:
if vc not in voicesToRemove: voicesToRemove.append(vc)
#
# restrict corpus ID by number of selected overlapping samples
# uses eval() to test against overlap number...
for vc in validVoices:
if self.simSelectRuleByCorpusId[vc] != None:
if not eval(
str(superimp.cnt['overlap'][timeInFrames]) +
self.simSelectRuleByCorpusId[vc]):
if vc not in voicesToRemove: voicesToRemove.append(vc)
#
# look to see if maxPercentTargetSegments has been exceeded
for vc in validVoices:
if self.data['cspInfo'][vc]['maxPercentTargetSegments'] == None:
continue
percentageSegmentsChosen = (float(
len(set(self.data['cspInfo'][vc]['selectedTargetSegments']))) /
float(self.totalNumberOfTargetSegments)
) * 100.
if percentageSegmentsChosen > self.data['cspInfo'][vc][
'maxPercentTargetSegments']:
voicesToRemove.append(vc)
#
# evaulate and set valid voices for instruments...
instruments.evaluate_voices(timeInFrames, validVoices)
instruments.setup_corpus_tests(timeInFrames)
###############################################
## NOW remove any voices that should be here ##
###############################################
for vc in voicesToRemove:
validVoices.remove(vc)
#####################################
## loop through each sound segment ##
#####################################
validSegments = []
for h in self.postLimitSegmentNormList:
# no voices that didn't pass through...
if h.voiceID not in validVoices: continue
#########################################
## remove based on clustering if asked ##
#########################################
if clusterMappingDict != {} and h.cluster != None and h.cluster != tgtclusterId:
continue
# if allowRepetition is False then test
if not h.allowRepetition and len(h.selectionTimes) > 0:
continue # skip this segment
# restiction in seconds per sample
if h.restrictRepetition != None and len(h.selectionTimes) > 0:
passme = True
for selectedTime in h.selectionTimes:
if abs(selectedTime - timeInSec) < h.restrictRepetition:
passme = False
if not passme: continue
# see if it overlaps too much (dependant on each segment's duration)
if h.restrictOverlaps != None:
max_look = min(timeInFrames + h.lengthInFrames,
len(superimp.cnt['cpsvc_overlap'][h.voiceID]))
maxOver = np.max(superimp.cnt['cpsvc_overlap'][h.voiceID]
[timeInFrames:max_look])
if maxOver >= h.restrictOverlaps: continue # skip this segment
# user string overlap restriction
for str, (counter,
limit) in superimp.cnt['corpusOverlapByString'].items():
if util.matchString(h.filename, str):
max_look = min(
timeInFrames + h.lengthInFrames,
len(superimp.cnt['corpusOverlapByString'][str][0]))
maxOver = np.max(superimp.cnt['corpusOverlapByString'][str]
[0][timeInFrames:max_look])
if maxOver >= limit: continue
################################################################################
## remove for instruments segment-specific restrictions i.e. pitch limit ##
## polyphony ##
################################################################################
if h.instrTag != None and not instruments.test_corpus_segment(
timeInFrames, h):
continue
# if we get here, add the segment
validSegments.append(h)
return validSegments
def __init__(self, corpusFromUserOptions,
corpusGlobalAttributesFromOptions,
restrictCorpusSelectionsByFilenameString, searchPaths,
AnalInterface, p):
self.preloadlist = []
self.preLimitSegmentList = []
self.postLimitSegmentNormList = []
self.selectedSegmentList = []
self.simSelectRuleByCorpusId = []
self.len = len(corpusFromUserOptions)
self.data = {}
self.data['lastVoice'] = None
self.data['totalLengthInSeconds'] = 0.
self.data['vcToCorpusName'] = []
self.data['postLimitSegmentDictVoice'] = {}
self.data['postLimitSegmentCount'] = 0
self.data['selectionTimeByVoice'] = {}
self.data['cspInfo'] = []
for cidx in range(len(corpusFromUserOptions)):
self.data['postLimitSegmentDictVoice'][cidx] = []
self.data['selectionTimeByVoice'][cidx] = []
# find any GLOBAL limitations the user has placed on the corpus
self.globalLimits = []
self.localLimits = []
if 'limit' in corpusGlobalAttributesFromOptions:
for stringy in corpusGlobalAttributesFromOptions['limit']:
self.globalLimits.append(
cpsLimit(stringy, range(len(corpusFromUserOptions)),
AnalInterface))
self.data['numberVoices'] = len(corpusFromUserOptions)
for cidx, cobj in enumerate(corpusFromUserOptions):
# add this voice
vcCnt = 0
cobj.name = util.verifyPath(cobj.name, AnalInterface.searchPaths)
cobj.voiceID = cidx
self.simSelectRuleByCorpusId.append(cobj.superimposeRule)
self.data['vcToCorpusName'].append(cobj.name)
for name, val in corpusGlobalAttributesFromOptions.items():
if name == 'limit': continue
setattr(cobj, name, val)
# add local limits
totalLimitList = []
for limitstr in cobj.limit:
limitObj = cpsLimit(limitstr, [cidx], AnalInterface)
self.localLimits.append(limitObj)
totalLimitList.append(limitObj)
# add global limits
totalLimitList.extend(
self.globalLimits) # from CORPUS_GLOBAL_ATTRIBUTES
# get segments/files list
timeList = []
if os.path.isdir(cobj.name): fileType = 'dir'
if os.path.isfile(cobj.name): fileType = 'file'
if os.path.islink(cobj.name): fileType = 'link'
# list of segmentation files?
if type(cobj.segmentationFile) in [tuple,
list]: # a list of seg files
cobj.segmentationFile = [
path.test(string, ops.SEARCH_PATHS)[1]
for string in cobj.segmentationFile
]
if fileType == 'file': # an audio file
##################
## input a FILE ## -- look for audacity-style txt label file
##################
times = []
if cobj.wholeFile:
times.append([0, None])
else:
cobj.segmentationFile = findSegmentationFile(
cobj.name, searchPaths[:], cobj.segmentationExtension,
cobj.wholeFile)
if not os.path.isfile(cobj.segmentationFile):
util.error(
'segmentation file',
"Cannot find segmentation file '%s'" %
cobj.segmentationFile)
sgs = util.readAudacityLabelFile(cobj.segmentationFile)
times.extend(sgs)
p.log(
"Using segments from segmentation file %s (%i segments)"
% (cobj.segmentationFile, len(sgs)))
for timeSeg in times:
timeList.append([cobj.name] + timeSeg)
cobj.numbSfFiles = 1
elif fileType == 'dir': # a directory
#######################
## input a DIRECTORY ##
#######################
files = util.getDirListOnlyExt(cobj.name, cobj.recursive,
AnalInterface.validSfExtensions)
cobj.segmentationFile = None # don't print it
for file in files:
segFileTest = findSegmentationFile(
file, searchPaths, cobj.segmentationExtension,
cobj.wholeFile)
if segFileTest != None and os.path.exists(segFileTest):
times = util.readAudacityLabelFile(segFileTest)
p.log(
"Using segments from segmentation file %s (%i segments)"
% (segFileTest, len(times)))
else:
times = [[0, None]]
for timeSeg in times:
timeList.append([file] + timeSeg)
cobj.numbSfFiles = len(files)
# reset counters...
segCount = 0
windowDist = descriptordata.hannWin(len(timeList) * 2)
# segment list
stringMatchingWithFullPaths = True
for idx in range(len(timeList)):
startSec = timeList[idx][1]
endSec = timeList[idx][2]
if cobj.start != None and startSec < cobj.start:
continue # skip it
if cobj.end != None and startSec > cobj.end:
continue # skip it
# matchSting: includeStr/excludeStr
if stringMatchingWithFullPaths:
stringMatchPath = os.path.abspath(timeList[idx][0])
else:
stringMatchPath = os.path.split(timeList[idx][0])[1]
if cobj.includeStr != None:
skip = True
if type(cobj.includeStr) not in [list, tuple]:
cobj.includeStr = [cobj.includeStr]
for test in cobj.includeStr:
if util.matchString(stringMatchPath,
test,
caseSensative=True):
skip = False
if skip: continue
if cobj.excludeStr != None:
skip = False
if type(cobj.excludeStr) not in [list, tuple]:
cobj.excludeStr = [cobj.excludeStr]
for test in cobj.excludeStr:
if util.matchString(stringMatchPath,
test,
caseSensative=True):
skip = True
if skip: continue
# minTime / maxTime
# matchTime: includeTimes/excludeTimes
if len(cobj.includeTimes) > 0:
skip = True
for timeTuple in cobj.includeTimes:
if startSec >= timeTuple[0] and endSec <= timeTuple[1]:
skip = False
if skip: continue
if len(cobj.excludeTimes) > 0:
skip = False
if type(cobj.excludeTimes) not in [list, tuple]:
cobj.excludeTimes = [cobj.excludeTimes
] # force it to be a list
for timeTuple in cobj.excludeTimes:
if startSec >= timeTuple[0] and startSec < timeTuple[1]:
skip = True
#print stringMatchPath, start, end, skip
if skip: continue
# see if there is any extra data from the segmentation file
if len(timeList[idx]) > 3:
segmentationfileData = ' '.join(timeList[idx][3:])
else:
segmentationfileData = None
# test if limitDur is set...
if cobj.limitDur != None:
if endSec != None and endSec - startSec > cobj.limitDur:
endSec = startSec + cobj.limitDur
# see which sf to map sound concatenation onto...
if cobj.concatFileName == None:
concatFileName = timeList[idx][0]
else:
concatFileName = cobj.concatFileName
# get any metadata
metadata = ''
for mstring, mstart, mstop in cobj.metadata:
if startSec >= mstart and startSec <= mstop:
metadata += mstring + ' '
# see if global RESTRICT_CORPUS_SELECT_PERCENTAGE_BY_STRING applies
maxPercentTargetSegmentsByString = None
for restrictStr, restrictVal in restrictCorpusSelectionsByFilenameString.items(
):
if util.matchString(timeList[idx][0], restrictStr):
maxPercentTargetSegmentsByString = restrictVal
self.preloadlist.append([
timeList[idx][0], timeList[idx][1], timeList[idx][2],
cobj.scaleDb, cobj.onsetLen, cobj.offsetLen,
cobj.envelopeSlope, AnalInterface, concatFileName,
cobj.name, cobj.voiceID, cobj.midiPitchMethod,
totalLimitList, cobj.pitchfilter, cobj.scaleDistance,
cobj.superimposeRule, cobj.transMethod, cobj.transQuantize,
cobj.allowRepetition, cobj.restrictInTime,
cobj.restrictOverlaps, cobj.restrictRepetition,
cobj.postSelectAmpBool, cobj.postSelectAmpMin,
cobj.postSelectAmpMax, cobj.postSelectAmpMethod,
segmentationfileData, metadata, cobj.clipDurationToTarget,
cobj.instrTag, cobj.instrParams
])
vcCnt += 1
self.data['cspInfo'].append({
'name': cobj.name,
'filehead': os.path.split(cobj.name)[1],
'segs': str(vcCnt),
'fileType': fileType,
'numbSfFiles': cobj.numbSfFiles,
'restrictInTime': cobj.restrictInTime,
'segFile': cobj.segmentationFile,
'restrictOverlaps': cobj.restrictOverlaps,
'scaleDb': cobj.scaleDb,
'maxPercentTargetSegments': cobj.maxPercentTargetSegments,
'selectedTargetSegments': [],
'instrTag': cobj.instrTag,
'instrParams': cobj.instrParams
})
###########################
## done with CORPUS loop ##
###########################
p.startPercentageBar(upperLabel="Evaluating CORPUS...",
total=len(self.preloadlist))
# in a seperate loop for printing...
for cidx, corpusSegParams in enumerate(self.preloadlist):
start = corpusSegParams[1]
stop = corpusSegParams[2]
if start == None: start = 0
if stop == None: stop = 100
p.percentageBarNext(lowerLabel="%s@%.2f-%.2f" %
(corpusSegParams[0], start, stop))
# make the obj
cpsSeg = sfsegment.corpusSegment(*corpusSegParams)
# add it to the list!
self.preLimitSegmentList.append(cpsSeg)
self.evaluatePreConcateLimitations()
self.evaluateCorpusPitchFilters()
self.finalizeSegmentNormList()
p.percentageBarClose(
txt="Read %i/%i segments (%.0f%%, %.2f min.)" %
(self.data['postLimitSegmentCount'], len(self.preLimitSegmentList),
self.data['postLimitSegmentCount'] /
float(len(self.preLimitSegmentList)) * 100.,
self.data['totalLengthInSeconds'] / 60.))
self.printConcateLimitations(p)