def run(self): excludedFiles = [ '.ipynb', '__pycache__', '.pyc', '.gitignore', '.DS_Store' ] for directoryPath, unused, fileNames in os.walk(self.docSourcePath): self.setupOutputDirectory( self.sourceToAutogenerated(directoryPath)) for fileName in fileNames: runIt = True for ex in excludedFiles: if fileName.endswith(ex): runIt = False if runIt is False: continue inputFilePath = os.path.join(directoryPath, fileName) outputFilePath = self.sourceToAutogenerated(inputFilePath) if (os.path.exists(outputFilePath) and os.path.getmtime(outputFilePath) > os.path.getmtime(inputFilePath)): print('\tSKIPPED {0}'.format( common.relativepath(outputFilePath))) else: shutil.copyfile(inputFilePath, outputFilePath) print('\tWROTE {0}'.format( common.relativepath(outputFilePath)))
def run(self): excludedFiles = [ '.ipynb', '__pycache__', '.pyc', '.gitignore', 'conf.py', '.DS_Store' ] for subPath in sorted(self.docSourcePath.rglob('*')): if subPath.is_dir(): self.setupOutputDirectory(self.sourceToAutogenerated(subPath)) continue runIt = True for ex in excludedFiles: if subPath.name.endswith(ex): runIt = False if runIt is False: continue outputFilePath = self.sourceToAutogenerated(subPath) if (outputFilePath.exists() and outputFilePath.stat().st_mtime > subPath.stat().st_mtime): print('\tSKIPPED {0}'.format( common.relativepath(outputFilePath))) else: shutil.copyfile(str(subPath), str(outputFilePath)) print('\tWROTE {0}'.format( common.relativepath(outputFilePath)))
def write(self, filePath, rst): # ''' Write ``rst`` (a unicode string) to ``filePath``, only overwriting an existing file if the content differs. ''' shouldWrite = True if os.path.exists(filePath): oldRst = common.readFileEncodingSafe(filePath, firstGuess='utf-8') if rst == oldRst: shouldWrite = False else: pass ## uncomment for help in figuring out why a file keeps being different... #import difflib #print(common.relativepath(filePath)) #print('\n'.join(difflib.ndiff(rst.split('\n'), oldRst.split('\n')))) if shouldWrite: with io.open(filePath, 'w', encoding='utf-8') as f: try: f.write(rst) except UnicodeEncodeError as uee: six.raise_from(DocumentationWritersException("Could not write %s with rst:\n%s" % (filePath, rst)), uee) print('\tWROTE {0}'.format(common.relativepath(filePath))) else: print('\tSKIPPED {0}'.format(common.relativepath(filePath)))
def write(self, filePath, rst): # ''' Write ``rst`` (a unicode string) to ``filePath``, a pathlib.Path() only overwriting an existing file if the content differs. ''' shouldWrite = True if filePath.exists(): oldRst = common.readFileEncodingSafe(filePath, firstGuess='utf-8') if rst == oldRst: shouldWrite = False else: pass ## uncomment for help in figuring out why a file keeps being different... #import difflib #print(common.relativepath(filePath)) #print('\n'.join(difflib.ndiff(rst.split('\n'), oldRst.split('\n')))) if shouldWrite: with filePath.open('w', encoding='utf-8') as f: try: f.write(rst) except UnicodeEncodeError as uee: raise DocumentationWritersException( "Could not write %s with rst:\n%s" % (filePath, rst)) from uee print('\tWROTE {0}'.format(common.relativepath(filePath))) else: print('\tSKIPPED {0}'.format(common.relativepath(filePath)))
def __iter__(self): import music21 rootFilesystemPath = music21.__path__[0] for directoryPath, directoryNames, fileNames in os.walk( rootFilesystemPath): directoryNamesToRemove = [] for directoryName in directoryNames: if directoryName in self._ignoredDirectoryNames: directoryNamesToRemove.append(directoryName) for directoryName in directoryNamesToRemove: directoryNames.remove(directoryName) if '__init__.py' in fileNames: strippedPath = directoryPath.partition(rootFilesystemPath)[2] pathParts = [x for x in strippedPath.split(os.path.sep) if x] pathParts.insert(0, 'music21') packagesystemPath = '.'.join(pathParts) try: module = __import__(packagesystemPath, fromlist=['*']) if getattr(module, '_DOC_IGNORE_MODULE_OR_PACKAGE', False): # Skip examining any other file or directory below # this directory. if self.verbose: print('\tIGNORED {0}/*'.format( common.relativepath(directoryPath))) directoryNames[:] = [] continue except ImportError: pass for fileName in fileNames: if fileName in self._ignoredFileNames: continue if not fileName.endswith('.py'): continue if fileName.startswith('_') and not fileName.startswith('__'): continue filePath = os.path.join(directoryPath, fileName) strippedPath = filePath.partition(rootFilesystemPath)[2] pathParts = [x for x in os.path.splitext( strippedPath)[0].split(os.path.sep)[1:] if x] pathParts = ['music21'] + pathParts packagesystemPath = '.'.join(pathParts) try: module = __import__(packagesystemPath, fromlist=['*']) if getattr(module, '_DOC_IGNORE_MODULE_OR_PACKAGE', False): if self.verbose: print('\tIGNORED {0}'.format( common.relativepath(filePath))) continue yield module except ImportError: pass raise StopIteration
def __iter__(self): rootFilesystemPath = str( common.getSourceFilePath()) # Remove str in Py3.6 for directoryPath, directoryNames, fileNames in os.walk( rootFilesystemPath): directoryNamesToRemove = [] for directoryName in sorted(directoryNames): if directoryName in self._ignoredDirectoryNames: directoryNamesToRemove.append(directoryName) for directoryName in directoryNamesToRemove: directoryNames.remove(directoryName) if '__init__.py' in fileNames: strippedPath = directoryPath.partition(rootFilesystemPath)[2] pathParts = [x for x in strippedPath.split(os.path.sep) if x] pathParts.insert(0, 'music21') packagesystemPath = '.'.join(pathParts) try: module = __import__(packagesystemPath, fromlist=['*']) if getattr(module, '_DOC_IGNORE_MODULE_OR_PACKAGE', False): # Skip examining any other file or directory below # this directory. if self.verbose: print('\tIGNORED {0}/*'.format( common.relativepath(directoryPath))) directoryNames[:] = [] continue except ImportError: pass for fileName in sorted(fileNames): if fileName in self._ignoredFileNames: continue if not fileName.endswith('.py'): continue if fileName.startswith('_') and not fileName.startswith('__'): continue filePath = os.path.join(directoryPath, fileName) strippedPath = filePath.partition(rootFilesystemPath)[2] pathParts = [ x for x in os.path.splitext(strippedPath)[0].split( os.path.sep)[1:] if x ] pathParts = ['music21'] + pathParts packagesystemPath = '.'.join(pathParts) try: module = __import__(packagesystemPath, fromlist=['*']) if getattr(module, '_DOC_IGNORE_MODULE_OR_PACKAGE', False): if self.verbose: print('\tIGNORED {0}'.format( common.relativepath(filePath))) continue yield module except ImportError: pass
def run(self): from music21 import documentation # @UnresolvedImport ipythonNotebookFilePaths = [x for x in documentation.IPythonNotebookIterator()] for ipythonNotebookFilePath in ipythonNotebookFilePaths: nbConvertReturnCode = self.convertOneNotebook(ipythonNotebookFilePath) if nbConvertReturnCode is True: self.cleanupNotebookAssets(ipythonNotebookFilePath) print('\tWROTE {0}'.format(common.relativepath( ipythonNotebookFilePath))) else: print('\tSKIPPED {0}'.format(common.relativepath( ipythonNotebookFilePath)))
def run(self): for ipythonNotebookFilePath in self.ipythonNotebookFilePaths: nbConvertReturnCode = self.convertOneNotebook(ipythonNotebookFilePath) if nbConvertReturnCode is True: self.cleanupNotebookAssets(ipythonNotebookFilePath) print('\tWROTE {0}'.format(common.relativepath( ipythonNotebookFilePath))) else: print('\tSKIPPED {0}'.format(common.relativepath( ipythonNotebookFilePath))) # do not print anything for skipped -checkpoint files self.writeIndexRst()
def __call__(self): from music21 import documentation # @UnresolvedImport ipythonNotebookFilePaths = [ x for x in documentation.IPythonNotebookIterator() ] for ipythonNotebookFilePath in ipythonNotebookFilePaths: nbConvertReturnCode = self._convertOneNotebook( ipythonNotebookFilePath) if nbConvertReturnCode is True: self._cleanupNotebookAssets(ipythonNotebookFilePath) print('\tWROTE {0}'.format( common.relativepath(ipythonNotebookFilePath))) else: print('\tSKIPPED {0}'.format( common.relativepath(ipythonNotebookFilePath)))
def write(self, filePath, rst): # ''' Write ``rst`` (a unicode string) to ``filePath``, only overwriting an existing file if the content differs. ''' shouldWrite = True if os.path.exists(filePath): oldRst = common.readFileEncodingSafe(filePath, firstGuess='utf-8') if rst == oldRst: shouldWrite = False if shouldWrite: with open(filePath, 'w') as f: f.write(rst) print('\tWROTE {0}'.format(common.relativepath(filePath))) else: print('\tSKIPPED {0}'.format(common.relativepath(filePath)))
def _parseNonOpus(self, parsedObject): from music21 import metadata try: corpusPath = metadata.MetadataBundle.corpusPathToKey( self.cleanFilePath) if parsedObject.metadata is not None: richMetadata = metadata.RichMetadata() richMetadata.merge(parsedObject.metadata) richMetadata.update(parsedObject) # update based on Stream environLocal.printDebug( 'updateMetadataCache: storing: {0}'.format(corpusPath)) metadataEntry = metadata.MetadataEntry( sourcePath=self.cleanFilePath, metadataPayload=richMetadata, ) self.results.append(metadataEntry) else: environLocal.printDebug( 'addFromPaths: got stream without metadata, ' 'creating stub: {0}'.format( common.relativepath(self.cleanFilePath))) metadataEntry = metadata.MetadataEntry( sourcePath=self.cleanFilePath, metadataPayload=None, ) self.results.append(metadataEntry) except Exception: environLocal.printDebug('Had a problem with extracting metadata ' 'for {0}, piece ignored'.format(self.filePath)) environLocal.printDebug(traceback.format_exc())
def cleanFilePath(self): corpusPath = os.path.abspath(common.getCorpusFilePath()) if self.filePath.startswith(corpusPath): cleanFilePath = common.relativepath(self.filePath, corpusPath) else: cleanFilePath = self.filePath return cleanFilePath
def parseNonOpus(self, parsedObject): from music21 import metadata try: corpusPath = metadata.bundles.MetadataBundle.corpusPathToKey( self.cleanFilePath) if parsedObject.metadata is not None: richMetadata = metadata.RichMetadata() richMetadata.merge(parsedObject.metadata) richMetadata.update(parsedObject) # update based on Stream environLocal.printDebug( f'updateMetadataCache: storing: {corpusPath}') metadataEntry = metadata.bundles.MetadataEntry( sourcePath=self.cleanFilePath, metadataPayload=richMetadata, corpusName=self.corpusName, ) self.results.append(metadataEntry) else: environLocal.printDebug( 'addFromPaths: got stream without metadata, ' 'creating stub: {0}'.format( common.relativepath(self.cleanFilePath))) metadataEntry = metadata.bundles.MetadataEntry( sourcePath=self.cleanFilePath, metadataPayload=None, corpusName=self.corpusName, ) self.results.append(metadataEntry) except Exception: # wide catch is fine. pylint: disable=broad-except environLocal.warn('Had a problem with extracting metadata ' 'for {0}, piece ignored'.format(self.filePath)) environLocal.warn(traceback.format_exc())
def write(self, filePath, rst): # ''' Write ``lines`` to ``filePath``, only overwriting an existing file if the content differs. ''' shouldWrite = True if os.path.exists(filePath): with open(filePath, 'r') as f: oldRst = f.read() if rst == oldRst: shouldWrite = False if shouldWrite: with open(filePath, 'w') as f: f.write(rst) print('\tWROTE {0}'.format(common.relativepath(filePath))) else: print('\tSKIPPED {0}'.format(common.relativepath(filePath)))
def write(self, filePath, rst): # ''' Write ``lines`` to ``filePath``, only overwriting an existing file if the content differs. ''' shouldWrite = True if os.path.exists(filePath): with open(filePath, 'r') as f: oldRst = f.read() if rst == oldRst: shouldWrite = False if shouldWrite: with open(filePath, 'w') as f: f.write(rst) print '\tWROTE {0}'.format(common.relativepath(filePath)) else: print '\tSKIPPED {0}'.format(common.relativepath(filePath))
def run(self): excludedFiles = ['.ipynb', '__pycache__', '.pyc', '.gitignore', '.DS_Store'] for directoryPath, unused, fileNames in os.walk(self.docSourcePath): self.setupOutputDirectory(self.sourceToAutogenerated(directoryPath)) for fileName in fileNames: runIt = True for ex in excludedFiles: if fileName.endswith(ex): runIt = False if runIt is False: continue inputFilePath = os.path.join(directoryPath, fileName) outputFilePath = self.sourceToAutogenerated(inputFilePath) if os.path.exists(outputFilePath) and os.path.getmtime(outputFilePath) > os.path.getmtime(inputFilePath): print('\tSKIPPED {0}'.format(common.relativepath(outputFilePath))) else: shutil.copyfile(inputFilePath, outputFilePath) print('\tWROTE {0}'.format(common.relativepath(outputFilePath)))
def write(self, filePath, rst): # ''' Write ``rst`` (a unicode string) to ``filePath``, only overwriting an existing file if the content differs. ''' shouldWrite = True if os.path.exists(filePath): oldRst = common.readFileEncodingSafe(filePath, firstGuess='utf-8') if rst == oldRst: shouldWrite = False if shouldWrite: with codecs.open(filePath, 'w', 'utf-8') as f: try: f.write(rst) except UnicodeEncodeError as uee: six.raise_from(DocumentationWritersException("Could not write %s with rst:\n%s" % (filePath, rst)), uee) print('\tWROTE {0}'.format(common.relativepath(filePath))) else: print('\tSKIPPED {0}'.format(common.relativepath(filePath)))
def run(self): excludedFiles = ['.ipynb', '__pycache__', '.pyc', '.gitignore', 'conf.py', '.DS_Store'] for subPath in sorted(self.docSourcePath.rglob('*')): if subPath.is_dir(): self.setupOutputDirectory(self.sourceToAutogenerated(subPath)) continue runIt = True for ex in excludedFiles: if subPath.name.endswith(ex): runIt = False if runIt is False: continue outputFilePath = self.sourceToAutogenerated(subPath) if (outputFilePath.exists() and outputFilePath.stat().st_mtime > subPath.stat().st_mtime): print('\tSKIPPED {0}'.format(common.relativepath(outputFilePath))) else: shutil.copyfile(str(subPath), str(outputFilePath)) print('\tWROTE {0}'.format(common.relativepath(outputFilePath)))
def write(self, filePath, rst): # ''' Write ``rst`` (a unicode string) to ``filePath``, only overwriting an existing file if the content differs. ''' shouldWrite = True if os.path.exists(filePath): oldRst = common.readFileEncodingSafe(filePath, firstGuess='utf-8') if rst == oldRst: shouldWrite = False if shouldWrite: with codecs.open(filePath, 'w', 'utf-8') as f: try: f.write(rst) except UnicodeEncodeError as uee: six.raise_from( DocumentationWritersException( "Could not write %s with rst:\n%s" % (filePath, rst)), uee) print('\tWROTE {0}'.format(common.relativepath(filePath))) else: print('\tSKIPPED {0}'.format(common.relativepath(filePath)))
def argRun(): parser = argparse.ArgumentParser(description='Run pylint on music21 according to style guide.') parser.add_argument('files', metavar='filename', type=str, nargs='*', help='Files to parse (default nearly all)') parser.add_argument('--strict', action='store_true', help='Run the file in strict mode') args = parser.parse_args() #print(args.files) #print(args.strict) files = args.files if args.files else None if files: sfp = common.getSourceFilePath() files = [common.relativepath(f, sfp) for f in files] main(files, args.strict)
def _main(target): from music21 import documentation # @UnresolvedImport documentationDirectoryPath = documentation.__path__[0] sourceDirectoryPath = os.path.join(documentationDirectoryPath, "source") buildDirectoryPath = os.path.join(documentationDirectoryPath, "build") doctreesDirectoryPath = os.path.join(buildDirectoryPath, "doctrees") buildDirectories = { "html": os.path.join(buildDirectoryPath, "html"), "latex": os.path.join(buildDirectoryPath, "latex"), "latexpdf": os.path.join(buildDirectoryPath, "latex"), } if target in buildDirectories: print "WRITING DOCUMENTATION FILES" documentation.ModuleReferenceReSTWriter()() documentation.CorpusReferenceReSTWriter()() documentation.IPythonNotebookReSTWriter()() sphinxOptions = ["sphinx"] sphinxOptions.extend(("-b", target)) sphinxOptions.extend(("-d", doctreesDirectoryPath)) sphinxOptions.append(sourceDirectoryPath) sphinxOptions.append(buildDirectories[target]) # sphinx.main() returns 0 on success, 1 on failure. # If the docs fail to build, we should not try to open a web browser. if sphinx.main(sphinxOptions): return if target == "html": launchPath = os.path.join(buildDirectories[target], "index.html") # TODO: Test launching web browsers under Windows. if launchPath.startswith("/"): launchPath = "file://" + launchPath webbrowser.open(launchPath) elif target == "clean": print "CLEANING AUTOGENERATED DOCUMENTATION" documentation.CorpusReferenceCleaner()() documentation.ModuleReferenceCleaner()() documentation.IPythonNotebookCleaner()() for name in os.listdir(buildDirectoryPath): if name.startswith("."): continue path = os.path.join(buildDirectoryPath, name) shutil.rmtree(path) print "\tCLEANED {0}".format(common.relativepath(path)) elif target == "help": _print_usage() else: print "Unsupported documentation target {!r}".format(target) print _print_usage()
def argRun(): parser = argparse.ArgumentParser( description='Run pylint on music21 according to style guide.') parser.add_argument('files', metavar='filename', type=str, nargs='*', help='Files to parse (default nearly all)') parser.add_argument('--strict', action='store_true', help='Run the file in strict mode') args = parser.parse_args() #print(args.files) #print(args.strict) files = args.files if args.files else None if files: sfp = common.getSourceFilePath() files = [common.relativepath(f, sfp) for f in files] main(files, args.strict)
def removeFile(self, filePath): if os.path.exists(filePath): print('\tCLEANED {0}'.format(common.relativepath(filePath))) os.remove(filePath)
def _main(target): from music21 import documentation # @UnresolvedImport documentationDirectoryPath = documentation.__path__[0] sourceDirectoryPath = os.path.join( documentationDirectoryPath, 'source', ) buildDirectoryPath = os.path.join( documentationDirectoryPath, 'build', ) doctreesDirectoryPath = os.path.join( buildDirectoryPath, 'doctrees', ) buildDirectories = { 'html': os.path.join( buildDirectoryPath, 'html', ), 'latex': os.path.join( buildDirectoryPath, 'latex', ), 'latexpdf': os.path.join( buildDirectoryPath, 'latex', ), } if target in buildDirectories: print('WRITING DOCUMENTATION FILES') documentation.ModuleReferenceReSTWriter()() documentation.CorpusReferenceReSTWriter()() documentation.IPythonNotebookReSTWriter()() sphinxOptions = ['sphinx'] sphinxOptions.extend(('-b', target)) sphinxOptions.extend(('-d', doctreesDirectoryPath)) sphinxOptions.append(sourceDirectoryPath) sphinxOptions.append(buildDirectories[target]) # sphinx.main() returns 0 on success, 1 on failure. # If the docs fail to build, we should not try to open a web browser. if sphinx.main(sphinxOptions): return if target == 'html': launchPath = os.path.join( buildDirectories[target], 'index.html', ) # TODO: Test launching web browsers under Windows. if launchPath.startswith('/'): launchPath = 'file://' + launchPath webbrowser.open(launchPath) elif target == 'clean': print('CLEANING AUTOGENERATED DOCUMENTATION') documentation.CorpusReferenceCleaner()() documentation.ModuleReferenceCleaner()() documentation.IPythonNotebookCleaner()() for name in os.listdir(buildDirectoryPath): if name.startswith('.'): continue path = os.path.join( buildDirectoryPath, name, ) shutil.rmtree(path) print('\tCLEANED {0}'.format(common.relativepath(path))) elif target == 'help': _print_usage() else: print('Unsupported documentation target {!r}\n'.format(target)) _print_usage()
def main(target): from music21 import documentation # @UnresolvedImport documentationDirectoryPath = documentation.__path__[0] sourceDirectoryPath = os.path.join( documentationDirectoryPath, 'source', ) buildDirectoryPath = os.path.join( documentationDirectoryPath, 'build', ) doctreesDirectoryPath = os.path.join( buildDirectoryPath, 'doctrees', ) buildDirectories = { 'html': os.path.join( buildDirectoryPath, 'html', ), 'latex': os.path.join( buildDirectoryPath, 'latex', ), 'latexpdf': os.path.join( buildDirectoryPath, 'latex', ), } if target in buildDirectories: print('WRITING DOCUMENTATION FILES') documentation.ModuleReferenceReSTWriter().run() documentation.CorpusReferenceReSTWriter().run() try: documentation.IPythonNotebookReSTWriter().run() except OSError: raise ImportError('IPythonNotebookReSTWriter crashed; most likely cause: no pandoc installed: https://github.com/jgm/pandoc/releases') sphinxOptions = ['sphinx'] sphinxOptions.extend(('-b', target)) sphinxOptions.extend(('-d', doctreesDirectoryPath)) sphinxOptions.append(sourceDirectoryPath) sphinxOptions.append(buildDirectories[target]) # sphinx.main() returns 0 on success, 1 on failure. # If the docs fail to build, we should not try to open a web browser. if sphinx.main(sphinxOptions): return if target == 'html': launchPath = os.path.join( buildDirectories[target], 'index.html', ) # TODO: Test launching web browsers under Windows. if launchPath.startswith('/'): launchPath = 'file://' + launchPath webbrowser.open(launchPath) elif target == 'clean': print('CLEANING AUTOGENERATED DOCUMENTATION') documentation.CorpusReferenceCleaner().run() documentation.ModuleReferenceCleaner().run() documentation.IPythonNotebookCleaner().run() for name in os.listdir(buildDirectoryPath): if name.startswith('.'): continue path = os.path.join( buildDirectoryPath, name, ) shutil.rmtree(path) print('\tCLEANED {0}'.format(common.relativepath(path))) elif target == 'help': _print_usage() else: print('Unsupported documentation target {!r}\n'.format(target)) _print_usage()