def getComposerDirectoryPath(self, composerName): ''' To be DEPRECATED Given the name of a composer, get the path to the top-level directory of that composer: >>> import os >>> from music21 import corpus >>> coreCorpus = corpus.corpora.CoreCorpus() >>> a = coreCorpus.getComposerDirectoryPath('ciconia') >>> a.endswith(os.path.join('corpus', os.sep, 'ciconia')) True >>> a = coreCorpus.getComposerDirectoryPath('bach') >>> a.endswith(os.path.join('corpus', os.sep, 'bach')) True >>> a = coreCorpus.getComposerDirectoryPath('handel') >>> a.endswith(os.path.join('corpus', os.sep, 'handel')) True ''' match = None for moduleName in sorted(os.listdir(common.getCorpusFilePath())): candidate = moduleName if composerName.lower() not in candidate.lower(): continue directory = os.path.join(common.getCorpusFilePath(), moduleName) if directory.lower().endswith(composerName.lower()): match = directory break return match
def cleanFilePath(self): corpusPath = os.path.abspath(common.getCorpusFilePath()) if self.filePath.startswith(corpusPath): cleanFilePath = common.relativepath(self.filePath, corpusPath) else: cleanFilePath = self.filePath return cleanFilePath
def md5OfCorpusFile(fileDir, scoreNumber=None): ''' returns the md5 hash of the text file contents. the file must be the full name of the corpus file >>> from music21.demos.bhadley.mrjobaws.awsutility import md5OfCorpusFile >>> a = md5OfCorpusFile('bach/bwv431.mxl') >>> a '3b8c4b8db4288c43efde44ddcdb4d8d2' >>> s = corpus.parse('bwv431') >>> s.corpusFilepath 'bach/bwv431.mxl' >>> b = md5OfCorpusFile(s.corpusFilepath) >>> b '3b8c4b8db4288c43efde44ddcdb4d8d2' >>> a == b True >>> md5OfCorpusFile('airdsAirs/book3.abc', '413') 'c1666c19d63fc0940f111008e2269f75.413' ''' corpusFP = common.getCorpusFilePath() / pathlib.Path(fileDir) with corpusFP.open('rb') as fileIn: md5 = common.getMd5(fileIn.read()) if scoreNumber: return md5 + '.' + scoreNumber else: return md5
def cleanFilePath(self): corpusPath = common.getCorpusFilePath() try: cleanFilePath = self.filePath.relative_to(corpusPath) except ValueError: cleanFilePath = self.filePath return cleanFilePath
def md5OfCorpusFile(fileDir, scoreNumber=None): """ returns the md5 hash of the text file contents. the file must be the full name of the corpus file >>> from music21.demos.bhadley.mrjobaws.awsutility import md5OfCorpusFile >>> a = md5OfCorpusFile('bach/bwv431.mxl') >>> a '3b8c4b8db4288c43efde44ddcdb4d8d2' >>> s = corpus.parse('bwv431') >>> s.corpusFilepath u'bach/bwv431.mxl' >>> b = md5OfCorpusFile(s.corpusFilepath) >>> b '3b8c4b8db4288c43efde44ddcdb4d8d2' >>> a == b True >>> md5OfCorpusFile('airdsAirs/book3.abc','413') 'c1666c19d63fc0940f111008e2269f75.413' """ corpusFP = common.getCorpusFilePath() fileIn = open(corpusFP + os.sep + fileDir, "rb") md5 = common.getMd5(fileIn.read()) if scoreNumber: return md5 + "." + scoreNumber else: return md5
def _addCorpusFilepath(streamObj, filePath): # metadata attribute added to store the file path, # for use later in identifying the score #if streamObj.metadata == None: # streamObj.insert(metadata.Metadata()) corpusFilePath = common.getCorpusFilePath() lenCFP = len(corpusFilePath) + len(os.sep) if filePath.startswith(corpusFilePath): fp2 = filePath[lenCFP:] ### corpus fix for windows dirsEtc = fp2.split(os.sep) fp3 = '/'.join(dirsEtc) streamObj.corpusFilepath = fp3 else: streamObj.corpusFilepath = filePath
def _addCorpusFilepathToStreamObject(streamObj, filePath): # metadata attribute added to store the file path, # for use later in identifying the score #if streamObj.metadata == None: # streamObj.insert(metadata.Metadata()) corpusFilePath = common.getCorpusFilePath() lenCFP = len(corpusFilePath) + len(os.sep) if filePath.startswith(corpusFilePath): fp2 = filePath[lenCFP:] ### corpus fix for windows dirsEtc = fp2.split(os.sep) fp3 = '/'.join(dirsEtc) streamObj.corpusFilepath = fp3 else: streamObj.corpusFilepath = filePath
def noCorpus(self): ''' Return True or False if this is a `corpus` or `noCoprus` distribution. >>> from music21 import corpus >>> corpus.corpora.CoreCorpus().noCorpus False ''' if CoreCorpus._noCorpus is None: # assume that there will always be a 'bach' dir for unused in common.getCorpusFilePath().iterdir(): CoreCorpus._noCorpus = False return False CoreCorpus._noCorpus = False return CoreCorpus._noCorpus
def noCorpus(self): ''' Return True or False if this is a `corpus` or `noCorpus` distribution. >>> from music21 import corpus >>> corpus.corpora.CoreCorpus().noCorpus False ''' if CoreCorpus._noCorpus is None: # assume that there will always be a 'bach' dir for unused in common.getCorpusFilePath().iterdir(): CoreCorpus._noCorpus = False return False CoreCorpus._noCorpus = False return CoreCorpus._noCorpus
def getPaths( self, fileExtensions=None, expandExtensions=True, ): ''' Get all paths in the core corpus that match a known extension, or an extension provided by an argument. If `expandExtensions` is True, a format for an extension, and related extensions, will replaced by all known input extensions. This is convenient when an input format might match for multiple extensions. >>> from music21 import corpus >>> coreCorpus = corpus.corpora.CoreCorpus() >>> corpusFilePaths = coreCorpus.getPaths() >>> 3000 < len(corpusFilePaths) < 4000 True >>> kernFilePaths = coreCorpus.getPaths('krn') >>> len(kernFilePaths) >= 500 True >>> abcFilePaths = coreCorpus.getPaths('abc') >>> len(abcFilePaths) >= 100 True ''' fileExtensions = self._translateExtensions( fileExtensions=fileExtensions, expandExtensions=expandExtensions, ) cacheKey = ('core', tuple(fileExtensions)) # not cached, fetch and reset if cacheKey not in Corpus._pathsCache: basePath = common.getCorpusFilePath() Corpus._pathsCache[cacheKey] = self._findPaths( basePath, fileExtensions, ) return Corpus._pathsCache[cacheKey]
def getPaths( self, fileExtensions=None, expandExtensions=True, ): ''' Get all paths in the core corpus that match a known extension, or an extenion provided by an argument. If `expandExtensions` is True, a format for an extension, and related extensions, will replaced by all known input extensions. This is convenient when an input format might match for multiple extensions. >>> from music21 import corpus >>> coreCorpus = corpus.corpora.CoreCorpus() >>> corpusFilePaths = coreCorpus.getPaths() >>> 3000 < len(corpusFilePaths) < 4000 True >>> kernFilePaths = coreCorpus.getPaths('krn') >>> len(kernFilePaths) >= 500 True >>> abcFilePaths = coreCorpus.getPaths('abc') >>> len(abcFilePaths) >= 100 True ''' fileExtensions = self._translateExtensions( fileExtensions=fileExtensions, expandExtensions=expandExtensions, ) cacheKey = ('core', tuple(fileExtensions)) # not cached, fetch and reset if cacheKey not in Corpus._pathsCache: basePath = common.getCorpusFilePath() Corpus._pathsCache[cacheKey] = self._findPaths( basePath, fileExtensions, ) return Corpus._pathsCache[cacheKey]
def validate(self): r''' Validate each metadata entry in a metadata bundle. If the entry represents a non-virtual corpus asset, test that its source path is locatable on disk. If not, remove the metadata entry from the metadata bundle. Currently (Dec 2014) there is one entry in the metadata bundle that has been removed, so calling validate (called from addFromPaths) results in 14418 instead of 14419 entries ''' timer = common.Timer() timer.start() environLocal.printDebug(['MetadataBundle: validating...']) invalidatedKeys = [] validatedPaths = set() for key, metadataEntry in self._metadataEntries.items(): # MetadataEntries for core corpus items use a relative path as # their source path, always starting with 'music21/corpus'. sourcePath = metadataEntry.sourcePath if sourcePath in validatedPaths: continue if sourcePath.startswith('http:'): validatedPaths.add(metadataEntry.sourcePath) continue if not os.path.isabs(sourcePath): sourcePath = os.path.abspath( os.path.join( common.getCorpusFilePath(), sourcePath, )) if not os.path.exists(sourcePath): invalidatedKeys.append(key) validatedPaths.add(metadataEntry.sourcePath) for key in invalidatedKeys: del (self._metadataEntries[key]) message = 'MetadataBundle: finished validating in {0} seconds.'.format( timer) environLocal.printDebug(message) return len(invalidatedKeys)
def cacheCoreMetadataMultiprocess(ipythonMod = None, stopAfter = None): '''The core cache is all locally-stored corpus files. ''' from music21 import corpus, metadata t = common.Timer() t.start() # store list of file paths that caused an error #fpError = [] mdb = metadata.MetadataBundle('core') pathsFull = corpus.getCorePaths() pathsShort = [] lenCorpusPath = len(common.getCorpusFilePath()) for i,p in enumerate(pathsFull): pathsShort.append(p[lenCorpusPath:]) if stopAfter is not None and i >= stopAfter: break environLocal.warn([ 'metadata cache: starting processing of paths:', len(pathsShort)]) #mdb.addFromPaths(paths[-3:]) # returns any paths that failed to load for i in range(0, len(pathsShort), 100): maxI = min(i+100, len(pathsShort)) pathsChunk = pathsShort[i:maxI] environLocal.warn('Starting multiprocessing with chunk %d, first is %s' % (i, pathsChunk[0])) allKeys = ipythonMod.map_async(cacheCoreMetadataMultiprocessHelper, pathsChunk) for thisKey in allKeys: for thisSubKey in thisKey: mdb.storage[thisSubKey[0]] = thisSubKey[1] #print mdb.storage mdb.write() # will use a default file path based on domain environLocal.warn(['cache: writing time:', t, 'md items:', len(mdb.storage)]) del mdb
def validate(self): r''' Validate each metadata entry in a metadata bundle. If the entry represents a non-virtual corpus asset, test that its source path is locatable on disk. If not, remove the metadata entry from the metadata bundle. Currently (Dec 2014) there is one entry in the metadata bundle that has been removed, so calling validate (called from addFromPaths) results in 14418 instead of 14419 entries ''' timer = common.Timer() timer.start() environLocal.printDebug(['MetadataBundle: validating...']) invalidatedKeys = [] validatedPaths = set() for key, metadataEntry in self._metadataEntries.items(): # MetadataEntries for core corpus items use a relative path as # their source path, always starting with 'music21/corpus'. sourcePath = metadataEntry.sourcePath if sourcePath in validatedPaths: continue if sourcePath.startswith('http:'): validatedPaths.add(metadataEntry.sourcePath) continue if not os.path.isabs(sourcePath): sourcePath = os.path.abspath(os.path.join( common.getCorpusFilePath(), sourcePath, )) if not os.path.exists(sourcePath): invalidatedKeys.append(key) validatedPaths.add(metadataEntry.sourcePath) for key in invalidatedKeys: del(self._metadataEntries[key]) message = 'MetadataBundle: finished validating in {0} seconds.'.format( timer) environLocal.printDebug(message) return len(invalidatedKeys)
def validate(self): r''' Validate each metadata entry in a metadata bundle. If the entry represents a non-virtual corpus asset, test that its source path is locatable on disk. If not, remove the metadata entry from the metadata bundle. ''' timer = common.Timer() timer.start() environLocal.printDebug(['MetadataBundle: validating...']) invalidatedKeys = [] validatedPaths = set() for key, metadataEntry in self._metadataEntries.items(): # MetadataEntries for core corpus items use a relative path as # their source path, always starting with 'music21/corpus'. sourcePath = metadataEntry.sourcePath if sourcePath in validatedPaths: continue if isinstance(sourcePath, str) and sourcePath.startswith('http:'): validatedPaths.add(metadataEntry.sourcePath) continue elif isinstance(sourcePath, str): sourcePath = pathlib.Path(sourcePath) if not sourcePath.is_absolute(): sourcePath = common.getCorpusFilePath() / sourcePath if not sourcePath.exists(): invalidatedKeys.append(key) validatedPaths.add(metadataEntry.sourcePath) for key in invalidatedKeys: del(self._metadataEntries[key]) message = 'MetadataBundle: finished validating in {0} seconds.'.format( timer) environLocal.printDebug(message) return len(invalidatedKeys)
def getSourcePath(self, streamObj): ''' Get a string of the path after the corpus for the piece...useful for searching on corpus items without proper composer data... >>> rmd = metadata.RichMetadata() >>> b = corpus.parse('bwv66.6') >>> rmd.getSourcePath(b) 'bach/bwv66.6.mxl' ''' if not hasattr(streamObj, 'filePath'): return '' # for some abc files... if not streamObj.filePath: return '' streamFp = streamObj.filePath if not isinstance(streamFp, pathlib.Path): streamFp = pathlib.Path(streamFp) try: relativePath = streamFp.relative_to(common.getCorpusFilePath()) return relativePath.as_posix() except ValueError: return streamFp.as_posix()
def _addCorpusFilepathToStreamObject(streamObj, filePath): ''' Adds an entry 'corpusFilepath' to the Stream object. TODO: this should work for non-core-corpora TODO: this should be in the metadata object TODO: this should set a pathlib.Path object ''' # metadata attribute added to store the file path, # for use later in identifying the score #if streamObj.metadata == None: # streamObj.insert(metadata.Metadata()) corpusFilePath = str(common.getCorpusFilePath()) lenCFP = len(corpusFilePath) + len(os.sep) filePath = str(filePath) if filePath.startswith(corpusFilePath): fp2 = filePath[lenCFP:] ### corpus fix for windows dirsEtc = fp2.split(os.sep) fp3 = '/'.join(dirsEtc) streamObj.corpusFilepath = fp3 else: streamObj.corpusFilepath = filePath
def _addCorpusFilepathToStreamObject(streamObj, filePath): ''' Adds an entry 'corpusFilepath' to the Stream object. TODO: this should work for non-core-corpora TODO: this should be in the metadata object TODO: this should set a pathlib.Path object ''' # metadata attribute added to store the file path, # for use later in identifying the score # if streamObj.metadata == None: # streamObj.insert(metadata.Metadata()) corpusFilePath = str(common.getCorpusFilePath()) lenCFP = len(corpusFilePath) + len(os.sep) filePath = str(filePath) if filePath.startswith(corpusFilePath): fp2 = filePath[lenCFP:] # corpus fix for windows dirsEtc = fp2.split(os.sep) fp3 = '/'.join(dirsEtc) streamObj.corpusFilepath = fp3 else: streamObj.corpusFilepath = filePath
def run(self, runWithEnviron=True): ''' Main code runner for testing. To set a new test, update the self.callTest attribute in __init__(). Note that the default of runWithEnviron imports music21.environment. That might skew results ''' suffix = '.png' # '.svg' outputFormat = suffix[1:] _MOD = "test.timeGraphs.py" if runWithEnviron: from music21 import environment environLocal = environment.Environment(_MOD) fp = environLocal.getTempFile(suffix) # manually get a temporary file else: import tempfile import os import sys if os.name in ['nt'] or sys.platform.startswith('win'): platform = 'win' else: platform = 'other' tempdir = os.path.join(tempfile.gettempdir(), 'music21') if platform != 'win': fd, fp = tempfile.mkstemp(dir=tempdir, suffix=suffix) if isinstance(fd, int): # on MacOS, fd returns an int, like 3, when this is called # in some context (specifically, programmatically in a # TestExternal class. the fp is still valid and works # TODO: this did not work on MacOS 10.6.8 w/ py 2.7 pass else: fd.close() else: tf = tempfile.NamedTemporaryFile(dir=tempdir, suffix=suffix) fp = tf.name tf.close() if self.includeList is not None: gf = pycallgraph.GlobbingFilter(include=self.includeList, exclude=self.excludeList) else: gf = pycallgraph.GlobbingFilter(exclude=self.excludeList) # create instance; will call setup routines ct = self.callTest() # start timer print('%s starting test' % _MOD) t = Timer() t.start() graphviz = pycallgraph.output.GraphvizOutput(output_file=fp) graphviz.tool = '/usr/local/bin/dot' config = pycallgraph.Config() config.trace_filter = gf from music21 import meter from music21 import note from music21 import converter from music21 import common beeth = common.getCorpusFilePath() + '/beethoven/opus133.mxl' #s = converter.parse(beeth, forceSource=True) #beeth = common.getCorpusFilePath() + '/bach/bwv66.6.mxl' s = converter.parse(beeth, forceSource=True) with pycallgraph.PyCallGraph(output=graphviz, config=config): #n = note.Note() #meter.TimeSignature('4/4') #ct.testFocus() # run routine pass print('elapsed time: %s' % t) # open the completed file print('file path: ' + fp) try: environLocal = environment.Environment(_MOD) environLocal.launch(outputFormat, fp) except NameError: pass