def getComposerDirectoryPath(self, composerName):
'''
To be DEPRECATED
Given the name of a composer, get the path to the top-level directory
of that composer:
>>> import os
>>> from music21 import corpus
>>> coreCorpus = corpus.corpora.CoreCorpus()
>>> a = coreCorpus.getComposerDirectoryPath('ciconia')
>>> a.endswith(os.path.join('corpus', os.sep, 'ciconia'))
True
>>> a = coreCorpus.getComposerDirectoryPath('bach')
>>> a.endswith(os.path.join('corpus', os.sep, 'bach'))
True
>>> a = coreCorpus.getComposerDirectoryPath('handel')
>>> a.endswith(os.path.join('corpus', os.sep, 'handel'))
True
'''
match = None
for moduleName in sorted(os.listdir(common.getCorpusFilePath())):
candidate = moduleName
if composerName.lower() not in candidate.lower():
continue
directory = os.path.join(common.getCorpusFilePath(), moduleName)
if directory.lower().endswith(composerName.lower()):
match = directory
break
return match
def cleanFilePath(self):
corpusPath = os.path.abspath(common.getCorpusFilePath())
if self.filePath.startswith(corpusPath):
cleanFilePath = common.relativepath(self.filePath, corpusPath)
else:
cleanFilePath = self.filePath
return cleanFilePath
def md5OfCorpusFile(fileDir, scoreNumber=None):
'''
returns the md5 hash of the text file contents. the file must be the full
name of the corpus file
>>> from music21.demos.bhadley.mrjobaws.awsutility import md5OfCorpusFile
>>> a = md5OfCorpusFile('bach/bwv431.mxl')
>>> a
'3b8c4b8db4288c43efde44ddcdb4d8d2'
>>> s = corpus.parse('bwv431')
>>> s.corpusFilepath
'bach/bwv431.mxl'
>>> b = md5OfCorpusFile(s.corpusFilepath)
>>> b
'3b8c4b8db4288c43efde44ddcdb4d8d2'
>>> a == b
True
>>> md5OfCorpusFile('airdsAirs/book3.abc', '413')
'c1666c19d63fc0940f111008e2269f75.413'
'''
corpusFP = common.getCorpusFilePath() / pathlib.Path(fileDir)
with corpusFP.open('rb') as fileIn:
md5 = common.getMd5(fileIn.read())
if scoreNumber:
return md5 + '.' + scoreNumber
else:
return md5
def cleanFilePath(self):
corpusPath = common.getCorpusFilePath()
try:
cleanFilePath = self.filePath.relative_to(corpusPath)
except ValueError:
cleanFilePath = self.filePath
return cleanFilePath
def cleanFilePath(self):
corpusPath = common.getCorpusFilePath()
try:
cleanFilePath = self.filePath.relative_to(corpusPath)
except ValueError:
cleanFilePath = self.filePath
return cleanFilePath
def md5OfCorpusFile(fileDir, scoreNumber=None):
"""
returns the md5 hash of the text file contents. the file must be the full
name of the corpus file
>>> from music21.demos.bhadley.mrjobaws.awsutility import md5OfCorpusFile
>>> a = md5OfCorpusFile('bach/bwv431.mxl')
>>> a
'3b8c4b8db4288c43efde44ddcdb4d8d2'
>>> s = corpus.parse('bwv431')
>>> s.corpusFilepath
u'bach/bwv431.mxl'
>>> b = md5OfCorpusFile(s.corpusFilepath)
>>> b
'3b8c4b8db4288c43efde44ddcdb4d8d2'
>>> a == b
True
>>> md5OfCorpusFile('airdsAirs/book3.abc','413')
'c1666c19d63fc0940f111008e2269f75.413'
"""
corpusFP = common.getCorpusFilePath()
fileIn = open(corpusFP + os.sep + fileDir, "rb")
md5 = common.getMd5(fileIn.read())
if scoreNumber:
return md5 + "." + scoreNumber
else:
return md5
def cleanFilePath(self):
corpusPath = os.path.abspath(common.getCorpusFilePath())
if self.filePath.startswith(corpusPath):
cleanFilePath = common.relativepath(self.filePath, corpusPath)
else:
cleanFilePath = self.filePath
return cleanFilePath
def _addCorpusFilepath(streamObj, filePath):
# metadata attribute added to store the file path,
# for use later in identifying the score
#if streamObj.metadata == None:
# streamObj.insert(metadata.Metadata())
corpusFilePath = common.getCorpusFilePath()
lenCFP = len(corpusFilePath) + len(os.sep)
if filePath.startswith(corpusFilePath):
fp2 = filePath[lenCFP:]
### corpus fix for windows
dirsEtc = fp2.split(os.sep)
fp3 = '/'.join(dirsEtc)
streamObj.corpusFilepath = fp3
else:
streamObj.corpusFilepath = filePath
def _addCorpusFilepathToStreamObject(streamObj, filePath):
# metadata attribute added to store the file path,
# for use later in identifying the score
#if streamObj.metadata == None:
# streamObj.insert(metadata.Metadata())
corpusFilePath = common.getCorpusFilePath()
lenCFP = len(corpusFilePath) + len(os.sep)
if filePath.startswith(corpusFilePath):
fp2 = filePath[lenCFP:]
### corpus fix for windows
dirsEtc = fp2.split(os.sep)
fp3 = '/'.join(dirsEtc)
streamObj.corpusFilepath = fp3
else:
streamObj.corpusFilepath = filePath
def noCorpus(self):
'''
Return True or False if this is a `corpus` or `noCoprus` distribution.
>>> from music21 import corpus
>>> corpus.corpora.CoreCorpus().noCorpus
False
'''
if CoreCorpus._noCorpus is None:
# assume that there will always be a 'bach' dir
for unused in common.getCorpusFilePath().iterdir():
CoreCorpus._noCorpus = False
return False
CoreCorpus._noCorpus = False
return CoreCorpus._noCorpus
def noCorpus(self):
'''
Return True or False if this is a `corpus` or `noCorpus` distribution.
>>> from music21 import corpus
>>> corpus.corpora.CoreCorpus().noCorpus
False
'''
if CoreCorpus._noCorpus is None:
# assume that there will always be a 'bach' dir
for unused in common.getCorpusFilePath().iterdir():
CoreCorpus._noCorpus = False
return False
CoreCorpus._noCorpus = False
return CoreCorpus._noCorpus
def getPaths(
self,
fileExtensions=None,
expandExtensions=True,
):
'''
Get all paths in the core corpus that match a known extension, or an
extension provided by an argument.
If `expandExtensions` is True, a format for an extension, and related
extensions, will replaced by all known input extensions.
This is convenient when an input format might match for multiple
extensions.
>>> from music21 import corpus
>>> coreCorpus = corpus.corpora.CoreCorpus()
>>> corpusFilePaths = coreCorpus.getPaths()
>>> 3000 < len(corpusFilePaths) < 4000
True
>>> kernFilePaths = coreCorpus.getPaths('krn')
>>> len(kernFilePaths) >= 500
True
>>> abcFilePaths = coreCorpus.getPaths('abc')
>>> len(abcFilePaths) >= 100
True
'''
fileExtensions = self._translateExtensions(
fileExtensions=fileExtensions,
expandExtensions=expandExtensions,
)
cacheKey = ('core', tuple(fileExtensions))
# not cached, fetch and reset
if cacheKey not in Corpus._pathsCache:
basePath = common.getCorpusFilePath()
Corpus._pathsCache[cacheKey] = self._findPaths(
basePath,
fileExtensions,
)
return Corpus._pathsCache[cacheKey]
def getPaths(
self,
fileExtensions=None,
expandExtensions=True,
):
'''
Get all paths in the core corpus that match a known extension, or an
extenion provided by an argument.
If `expandExtensions` is True, a format for an extension, and related
extensions, will replaced by all known input extensions.
This is convenient when an input format might match for multiple
extensions.
>>> from music21 import corpus
>>> coreCorpus = corpus.corpora.CoreCorpus()
>>> corpusFilePaths = coreCorpus.getPaths()
>>> 3000 < len(corpusFilePaths) < 4000
True
>>> kernFilePaths = coreCorpus.getPaths('krn')
>>> len(kernFilePaths) >= 500
True
>>> abcFilePaths = coreCorpus.getPaths('abc')
>>> len(abcFilePaths) >= 100
True
'''
fileExtensions = self._translateExtensions(
fileExtensions=fileExtensions,
expandExtensions=expandExtensions,
)
cacheKey = ('core', tuple(fileExtensions))
# not cached, fetch and reset
if cacheKey not in Corpus._pathsCache:
basePath = common.getCorpusFilePath()
Corpus._pathsCache[cacheKey] = self._findPaths(
basePath,
fileExtensions,
)
return Corpus._pathsCache[cacheKey]
def validate(self):
r'''
Validate each metadata entry in a metadata bundle.
If the entry represents a non-virtual corpus asset, test that its
source path is locatable on disk. If not, remove the metadata entry
from the metadata bundle.
Currently (Dec 2014) there is one entry in the metadata bundle that
has been removed, so calling validate (called from addFromPaths) results in
14418 instead of 14419 entries
'''
timer = common.Timer()
timer.start()
environLocal.printDebug(['MetadataBundle: validating...'])
invalidatedKeys = []
validatedPaths = set()
for key, metadataEntry in self._metadataEntries.items():
# MetadataEntries for core corpus items use a relative path as
# their source path, always starting with 'music21/corpus'.
sourcePath = metadataEntry.sourcePath
if sourcePath in validatedPaths:
continue
if sourcePath.startswith('http:'):
validatedPaths.add(metadataEntry.sourcePath)
continue
if not os.path.isabs(sourcePath):
sourcePath = os.path.abspath(
os.path.join(
common.getCorpusFilePath(),
sourcePath,
))
if not os.path.exists(sourcePath):
invalidatedKeys.append(key)
validatedPaths.add(metadataEntry.sourcePath)
for key in invalidatedKeys:
del (self._metadataEntries[key])
message = 'MetadataBundle: finished validating in {0} seconds.'.format(
timer)
environLocal.printDebug(message)
return len(invalidatedKeys)
def cacheCoreMetadataMultiprocess(ipythonMod = None, stopAfter = None):
'''The core cache is all locally-stored corpus files.
'''
from music21 import corpus, metadata
t = common.Timer()
t.start()
# store list of file paths that caused an error
#fpError = []
mdb = metadata.MetadataBundle('core')
pathsFull = corpus.getCorePaths()
pathsShort = []
lenCorpusPath = len(common.getCorpusFilePath())
for i,p in enumerate(pathsFull):
pathsShort.append(p[lenCorpusPath:])
if stopAfter is not None and i >= stopAfter:
break
environLocal.warn([
'metadata cache: starting processing of paths:', len(pathsShort)])
#mdb.addFromPaths(paths[-3:])
# returns any paths that failed to load
for i in range(0, len(pathsShort), 100):
maxI = min(i+100, len(pathsShort))
pathsChunk = pathsShort[i:maxI]
environLocal.warn('Starting multiprocessing with chunk %d, first is %s' % (i, pathsChunk[0]))
allKeys = ipythonMod.map_async(cacheCoreMetadataMultiprocessHelper, pathsChunk)
for thisKey in allKeys:
for thisSubKey in thisKey:
mdb.storage[thisSubKey[0]] = thisSubKey[1]
#print mdb.storage
mdb.write() # will use a default file path based on domain
environLocal.warn(['cache: writing time:', t, 'md items:', len(mdb.storage)])
del mdb
def validate(self):
r'''
Validate each metadata entry in a metadata bundle.
If the entry represents a non-virtual corpus asset, test that its
source path is locatable on disk. If not, remove the metadata entry
from the metadata bundle.
Currently (Dec 2014) there is one entry in the metadata bundle that
has been removed, so calling validate (called from addFromPaths) results in
14418 instead of 14419 entries
'''
timer = common.Timer()
timer.start()
environLocal.printDebug(['MetadataBundle: validating...'])
invalidatedKeys = []
validatedPaths = set()
for key, metadataEntry in self._metadataEntries.items():
# MetadataEntries for core corpus items use a relative path as
# their source path, always starting with 'music21/corpus'.
sourcePath = metadataEntry.sourcePath
if sourcePath in validatedPaths:
continue
if sourcePath.startswith('http:'):
validatedPaths.add(metadataEntry.sourcePath)
continue
if not os.path.isabs(sourcePath):
sourcePath = os.path.abspath(os.path.join(
common.getCorpusFilePath(),
sourcePath,
))
if not os.path.exists(sourcePath):
invalidatedKeys.append(key)
validatedPaths.add(metadataEntry.sourcePath)
for key in invalidatedKeys:
del(self._metadataEntries[key])
message = 'MetadataBundle: finished validating in {0} seconds.'.format(
timer)
environLocal.printDebug(message)
return len(invalidatedKeys)
def validate(self):
r'''
Validate each metadata entry in a metadata bundle.
If the entry represents a non-virtual corpus asset, test that its
source path is locatable on disk. If not, remove the metadata entry
from the metadata bundle.
'''
timer = common.Timer()
timer.start()
environLocal.printDebug(['MetadataBundle: validating...'])
invalidatedKeys = []
validatedPaths = set()
for key, metadataEntry in self._metadataEntries.items():
# MetadataEntries for core corpus items use a relative path as
# their source path, always starting with 'music21/corpus'.
sourcePath = metadataEntry.sourcePath
if sourcePath in validatedPaths:
continue
if isinstance(sourcePath, str) and sourcePath.startswith('http:'):
validatedPaths.add(metadataEntry.sourcePath)
continue
elif isinstance(sourcePath, str):
sourcePath = pathlib.Path(sourcePath)
if not sourcePath.is_absolute():
sourcePath = common.getCorpusFilePath() / sourcePath
if not sourcePath.exists():
invalidatedKeys.append(key)
validatedPaths.add(metadataEntry.sourcePath)
for key in invalidatedKeys:
del(self._metadataEntries[key])
message = 'MetadataBundle: finished validating in {0} seconds.'.format(
timer)
environLocal.printDebug(message)
return len(invalidatedKeys)
def validate(self):
r'''
Validate each metadata entry in a metadata bundle.
If the entry represents a non-virtual corpus asset, test that its
source path is locatable on disk. If not, remove the metadata entry
from the metadata bundle.
'''
timer = common.Timer()
timer.start()
environLocal.printDebug(['MetadataBundle: validating...'])
invalidatedKeys = []
validatedPaths = set()
for key, metadataEntry in self._metadataEntries.items():
# MetadataEntries for core corpus items use a relative path as
# their source path, always starting with 'music21/corpus'.
sourcePath = metadataEntry.sourcePath
if sourcePath in validatedPaths:
continue
if isinstance(sourcePath, str) and sourcePath.startswith('http:'):
validatedPaths.add(metadataEntry.sourcePath)
continue
elif isinstance(sourcePath, str):
sourcePath = pathlib.Path(sourcePath)
if not sourcePath.is_absolute():
sourcePath = common.getCorpusFilePath() / sourcePath
if not sourcePath.exists():
invalidatedKeys.append(key)
validatedPaths.add(metadataEntry.sourcePath)
for key in invalidatedKeys:
del(self._metadataEntries[key])
message = 'MetadataBundle: finished validating in {0} seconds.'.format(
timer)
environLocal.printDebug(message)
return len(invalidatedKeys)
def getSourcePath(self, streamObj):
'''
Get a string of the path after the corpus for the piece...useful for
searching on corpus items without proper composer data...
>>> rmd = metadata.RichMetadata()
>>> b = corpus.parse('bwv66.6')
>>> rmd.getSourcePath(b)
'bach/bwv66.6.mxl'
'''
if not hasattr(streamObj, 'filePath'):
return '' # for some abc files...
if not streamObj.filePath:
return ''
streamFp = streamObj.filePath
if not isinstance(streamFp, pathlib.Path):
streamFp = pathlib.Path(streamFp)
try:
relativePath = streamFp.relative_to(common.getCorpusFilePath())
return relativePath.as_posix()
except ValueError:
return streamFp.as_posix()
def getSourcePath(self, streamObj):
'''
Get a string of the path after the corpus for the piece...useful for
searching on corpus items without proper composer data...
>>> rmd = metadata.RichMetadata()
>>> b = corpus.parse('bwv66.6')
>>> rmd.getSourcePath(b)
'bach/bwv66.6.mxl'
'''
if not hasattr(streamObj, 'filePath'):
return '' # for some abc files...
if not streamObj.filePath:
return ''
streamFp = streamObj.filePath
if not isinstance(streamFp, pathlib.Path):
streamFp = pathlib.Path(streamFp)
try:
relativePath = streamFp.relative_to(common.getCorpusFilePath())
return relativePath.as_posix()
except ValueError:
return streamFp.as_posix()
def _addCorpusFilepathToStreamObject(streamObj, filePath):
'''
Adds an entry 'corpusFilepath' to the Stream object.
TODO: this should work for non-core-corpora
TODO: this should be in the metadata object
TODO: this should set a pathlib.Path object
'''
# metadata attribute added to store the file path,
# for use later in identifying the score
#if streamObj.metadata == None:
# streamObj.insert(metadata.Metadata())
corpusFilePath = str(common.getCorpusFilePath())
lenCFP = len(corpusFilePath) + len(os.sep)
filePath = str(filePath)
if filePath.startswith(corpusFilePath):
fp2 = filePath[lenCFP:]
### corpus fix for windows
dirsEtc = fp2.split(os.sep)
fp3 = '/'.join(dirsEtc)
streamObj.corpusFilepath = fp3
else:
streamObj.corpusFilepath = filePath
def _addCorpusFilepathToStreamObject(streamObj, filePath):
'''
Adds an entry 'corpusFilepath' to the Stream object.
TODO: this should work for non-core-corpora
TODO: this should be in the metadata object
TODO: this should set a pathlib.Path object
'''
# metadata attribute added to store the file path,
# for use later in identifying the score
# if streamObj.metadata == None:
# streamObj.insert(metadata.Metadata())
corpusFilePath = str(common.getCorpusFilePath())
lenCFP = len(corpusFilePath) + len(os.sep)
filePath = str(filePath)
if filePath.startswith(corpusFilePath):
fp2 = filePath[lenCFP:]
# corpus fix for windows
dirsEtc = fp2.split(os.sep)
fp3 = '/'.join(dirsEtc)
streamObj.corpusFilepath = fp3
else:
streamObj.corpusFilepath = filePath
def run(self, runWithEnviron=True):
'''
Main code runner for testing. To set a new test, update the self.callTest attribute in __init__().
Note that the default of runWithEnviron imports music21.environment. That might
skew results
'''
suffix = '.png' # '.svg'
outputFormat = suffix[1:]
_MOD = "test.timeGraphs.py"
if runWithEnviron:
from music21 import environment
environLocal = environment.Environment(_MOD)
fp = environLocal.getTempFile(suffix)
# manually get a temporary file
else:
import tempfile
import os
import sys
if os.name in ['nt'] or sys.platform.startswith('win'):
platform = 'win'
else:
platform = 'other'
tempdir = os.path.join(tempfile.gettempdir(), 'music21')
if platform != 'win':
fd, fp = tempfile.mkstemp(dir=tempdir, suffix=suffix)
if isinstance(fd, int):
# on MacOS, fd returns an int, like 3, when this is called
# in some context (specifically, programmatically in a
# TestExternal class. the fp is still valid and works
# TODO: this did not work on MacOS 10.6.8 w/ py 2.7
pass
else:
fd.close()
else:
tf = tempfile.NamedTemporaryFile(dir=tempdir, suffix=suffix)
fp = tf.name
tf.close()
if self.includeList is not None:
gf = pycallgraph.GlobbingFilter(include=self.includeList,
exclude=self.excludeList)
else:
gf = pycallgraph.GlobbingFilter(exclude=self.excludeList)
# create instance; will call setup routines
ct = self.callTest()
# start timer
print('%s starting test' % _MOD)
t = Timer()
t.start()
graphviz = pycallgraph.output.GraphvizOutput(output_file=fp)
graphviz.tool = '/usr/local/bin/dot'
config = pycallgraph.Config()
config.trace_filter = gf
from music21 import meter
from music21 import note
from music21 import converter
from music21 import common
beeth = common.getCorpusFilePath() + '/beethoven/opus133.mxl'
#s = converter.parse(beeth, forceSource=True)
#beeth = common.getCorpusFilePath() + '/bach/bwv66.6.mxl'
s = converter.parse(beeth, forceSource=True)
with pycallgraph.PyCallGraph(output=graphviz, config=config):
#n = note.Note()
#meter.TimeSignature('4/4')
#ct.testFocus() # run routine
pass
print('elapsed time: %s' % t)
# open the completed file
print('file path: ' + fp)
try:
environLocal = environment.Environment(_MOD)
environLocal.launch(outputFormat, fp)
except NameError:
pass
def run(self, runWithEnviron=True):
'''
Main code runner for testing. To set a new test, update the self.callTest attribute in __init__().
Note that the default of runWithEnviron imports music21.environment. That might
skew results
'''
suffix = '.png' # '.svg'
outputFormat = suffix[1:]
_MOD = "test.timeGraphs.py"
if runWithEnviron:
from music21 import environment
environLocal = environment.Environment(_MOD)
fp = environLocal.getTempFile(suffix)
# manually get a temporary file
else:
import tempfile
import os
import sys
if os.name in ['nt'] or sys.platform.startswith('win'):
platform = 'win'
else:
platform = 'other'
tempdir = os.path.join(tempfile.gettempdir(), 'music21')
if platform != 'win':
fd, fp = tempfile.mkstemp(dir=tempdir, suffix=suffix)
if isinstance(fd, int):
# on MacOS, fd returns an int, like 3, when this is called
# in some context (specifically, programmatically in a
# TestExternal class. the fp is still valid and works
# TODO: this did not work on MacOS 10.6.8 w/ py 2.7
pass
else:
fd.close()
else:
tf = tempfile.NamedTemporaryFile(dir=tempdir, suffix=suffix)
fp = tf.name
tf.close()
if self.includeList is not None:
gf = pycallgraph.GlobbingFilter(include=self.includeList, exclude=self.excludeList)
else:
gf = pycallgraph.GlobbingFilter(exclude=self.excludeList)
# create instance; will call setup routines
ct = self.callTest()
# start timer
print('%s starting test' % _MOD)
t = Timer()
t.start()
graphviz = pycallgraph.output.GraphvizOutput(output_file=fp)
graphviz.tool = '/usr/local/bin/dot'
config = pycallgraph.Config()
config.trace_filter = gf
from music21 import meter
from music21 import note
from music21 import converter
from music21 import common
beeth = common.getCorpusFilePath() + '/beethoven/opus133.mxl'
#s = converter.parse(beeth, forceSource=True)
#beeth = common.getCorpusFilePath() + '/bach/bwv66.6.mxl'
s = converter.parse(beeth, forceSource=True)
with pycallgraph.PyCallGraph(output=graphviz, config=config):
#n = note.Note()
#meter.TimeSignature('4/4')
#ct.testFocus() # run routine
pass
print('elapsed time: %s' % t)
# open the completed file
print('file path: ' + fp)
try:
environLocal = environment.Environment(_MOD)
environLocal.launch(outputFormat, fp)
except NameError:
pass
