Exemple #1
0
    def walk(self):
        '''
        Get all the modules in reverse order, storing them in self.modulePaths
        '''
        def manyCoreSortFunc(name):
            '''
            for many core systems, like the MacPro, running slowest modules first
            helps there be fewer idle cores later
            '''
            name = name[len(self.dirParent) + 1:]
            name = name.replace('.py', '')
            return (name in self.slowModules, name)

        # the results of this are stored in self.curFiles, self.dirList
        for dirPath, unused_dirNames, filenames in os.walk(self.dirParent):
            self._visitFunc(None, dirPath, filenames)

        if common.cpus() > 4:  # @UndefinedVariable
            self.modulePaths.sort(key=manyCoreSortFunc)
        else:
            self.modulePaths.sort()

        # for p in self.modulePaths:
        #    print(p)
        self.modulePaths.reverse()
Exemple #2
0
    def walk(self):
        '''
        Get all the modules in reverse order, storing them in self.modulePaths
        '''
        def manyCoreSortFunc(name):
            '''
            for many core systems, like the MacPro, running slowest modules first
            helps there be fewer idle cores later 
            '''
            name = name[len(self.dirParent) + 1:]
            name = name.replace('.py', '')
            return (name in self.slowModules, name)

        # the results of this are stored in self.curFiles, self.dirList
        for dirpath, unused_dirnames, filenames in os.walk(self.dirParent):
            self._visitFunc(None, dirpath, filenames)

        if common.cpus() > 4:# @UndefinedVariable
            self.modulePaths.sort(key=manyCoreSortFunc)
        else:
            self.modulePaths.sort()

        #for p in self.modulePaths:
        #    print p# self.modulePaths
        self.modulePaths.reverse()
Exemple #3
0
    def walk(self):
        '''
        Get all the modules in reverse order, storing them in self.modulePaths
        '''
        def manyCoreSortFunc(name):
            '''
            for many core systems, like the MacPro, running slowest modules first
            helps there be fewer idle cores later
            '''
            name = name[len(self.dirParent) + 1:]
            name = name.replace('.py', '')
            return (name in self.slowModules, name)

        # the results of this are stored in self.curFiles, self.dirList
        for dirPath, unused_dirNames, filenames in os.walk(self.dirParent):
            self._visitFunc(None, dirPath, filenames)

        if common.cpus() > 4:  # @UndefinedVariable
            self.modulePaths.sort(key=manyCoreSortFunc)
        else:
            self.modulePaths.sort()

        # I don't know why we do this, but it happened this way in an early m21 version,
        # and it's just tradition now. It'd wig me out
        # if I ever didn't see them in reverse alphabetical order.
        self.modulePaths.reverse()
Exemple #4
0
 def __init__(self, command='html'):
     self.useMultiprocessing = True
     self.cpus_to_use = common.cpus()
     if self.cpus_to_use == 1:
         self.useMultiprocessing = False
     self.useMultiprocessing = False # too unstable still
     self.documentationDirectoryPath = None
     self.autogenDirectoryPath = None
     self.buildDirectoryPath = None
     self.doctreesDirectoryPath = None
     self.command = command
     self.getPaths()
Exemple #5
0
 def __init__(self, command='html'):
     self.useMultiprocessing = True
     self.cpus_to_use = common.cpus()
     if self.cpus_to_use == 1:
         self.useMultiprocessing = False
     self.useMultiprocessing = False  # too unstable still
     self.documentationDirectoryPath = None
     self.autogenDirectoryPath = None
     self.buildDirectoryPath = None
     self.doctreesDirectoryPath = None
     self.command = command
     self.getPaths()
Exemple #6
0
    def process_parallel(jobs, processCount=None):
        '''
        Process jobs in parallel, with `processCount` processes.

        If `processCount` is none, use 1 fewer process than the number of
        available cores.
        
        jobs is a list of :class:`~music21.metadata.MetadataCachingJob` objects.
        
        '''
        processCount = processCount or common.cpus()  # @UndefinedVariable
        if processCount < 1:
            processCount = 1
        remainingJobs = len(jobs)
        if processCount > remainingJobs:  # do not start more processes than jobs...
            processCount = remainingJobs

        environLocal.printDebug(
            'Processing {0} jobs in parallel, with {1} processes.'.format(
                remainingJobs, processCount))
        results = []
        job_queue = multiprocessing.JoinableQueue()  # @UndefinedVariable
        result_queue = multiprocessing.Queue()  # @UndefinedVariable
        workers = [
            WorkerProcess(job_queue, result_queue) for _ in range(processCount)
        ]
        for worker in workers:
            worker.start()
        if jobs:
            for job in jobs:
                job_queue.put(
                    pickle.dumps(job, protocol=pickle.HIGHEST_PROTOCOL))
            for unused_jobCounter in range(len(jobs)):
                job = pickle.loads(result_queue.get())
                results = job.getResults()
                errors = job.getErrors()
                remainingJobs -= 1
                yield {
                    'metadataEntries': results,
                    'errors': errors,
                    'filePath': job.filePath,
                    'remainingJobs': remainingJobs,
                }
        for worker in workers:
            job_queue.put(None)
        job_queue.join()
        result_queue.close()
        job_queue.close()
        for worker in workers:
            worker.join()
        raise StopIteration
Exemple #7
0
    def process_parallel(jobs, processCount=None):
        '''
        Process jobs in parallel, with `processCount` processes.

        If `processCount` is none, use 1 fewer process than the number of
        available cores.

        jobs is a list of :class:`~music21.metadata.MetadataCachingJob` objects.

        '''
        processCount = processCount or common.cpus()
        processCount = max(processCount, 1)
        # do not start more processes than jobs...
        remainingJobs = len(jobs)
        processCount = min(processCount, remainingJobs)

        environLocal.printDebug(
            f'Processing {remainingJobs} jobs in parallel, with {processCount} processes.'
        )
        results = []
        job_queue = multiprocessing.JoinableQueue()
        result_queue = multiprocessing.Queue()
        workers = [
            WorkerProcess(job_queue, result_queue) for _ in range(processCount)
        ]
        for worker in workers:
            worker.start()
        if jobs:
            for job in jobs:
                job_queue.put(pickle.dumps(
                    job))  # do not use highest protocol to generate.
            for unused_jobCounter in range(len(jobs)):
                job = pickle.loads(result_queue.get())
                results = job.getResults()
                errors = job.getErrors()
                remainingJobs -= 1
                yield {
                    'metadataEntries': results,
                    'errors': errors,
                    'filePath': job.filePath,
                    'remainingJobs': remainingJobs,
                }
        for worker in workers:
            job_queue.put(None)
        job_queue.join()
        result_queue.close()
        job_queue.close()
        for worker in workers:
            worker.join()
Exemple #8
0
    def process_parallel(jobs, processCount=None):
        '''
        Process jobs in parallel, with `processCount` processes.

        If `processCount` is none, use 1 fewer process than the number of
        available cores.
        
        jobs is a list of :class:`~music21.metadata.MetadataCachingJob` objects.
        
        '''
        processCount = processCount or common.cpus()  # @UndefinedVariable
        if processCount < 1:
            processCount = 1
        remainingJobs = len(jobs)
        if processCount > remainingJobs: # do not start more processes than jobs...
            processCount = remainingJobs
            
        environLocal.printDebug(
            'Processing {0} jobs in parallel, with {1} processes.'.format(
                remainingJobs, processCount))
        results = []
        job_queue = multiprocessing.JoinableQueue() # @UndefinedVariable
        result_queue = multiprocessing.Queue() # @UndefinedVariable
        workers = [WorkerProcess(job_queue, result_queue)
            for _ in range(processCount)]
        for worker in workers:
            worker.start()
        if jobs:
            for job in jobs:
                job_queue.put(pickle.dumps(job, protocol=pickle.HIGHEST_PROTOCOL))
            for unused_jobCounter in range(len(jobs)):
                job = pickle.loads(result_queue.get())
                results = job.getResults()
                errors = job.getErrors()
                remainingJobs -= 1
                yield {
                    'metadataEntries': results,
                    'errors': errors,
                    'filePath': job.filePath,
                    'remainingJobs': remainingJobs,
                    }
        for worker in workers:
            job_queue.put(None)
        job_queue.join()
        result_queue.close()
        job_queue.close()
        for worker in workers:
            worker.join()
        raise StopIteration
Exemple #9
0
def mainPoolRunner(
        testGroup=('test', ), restoreEnvironmentDefaults=False, leaveOut=1):
    '''
    Run all tests. Group can be test and/or external
    '''
    normalStdError = sys.stderr

    timeStart = time.time()
    poolSize = common.cpus()

    print(
        'Creating %d processes for multiprocessing (omitting %d processors)' %
        (poolSize, leaveOut))

    modGather = commonTest.ModuleGather(useExtended=True)

    maxTimeout = 200
    pathsToRun = modGather.modulePaths  # [30:60]

    pool = multiprocessing.Pool(processes=poolSize)  # @UndefinedVariable # pylint: disable=not-callable

    # imap returns the results as they are completed.  Since the number of files is small,
    # the overhead of returning is outweighed by the positive aspect of getting results immediately
    # unordered says that results can RETURN in any order; not that they'd be pooled out in any
    # order.
    res = pool.imap_unordered(runOneModuleWithoutImp,
                              ((modGather, fp) for fp in pathsToRun))

    continueIt = True
    timeouts = 0
    eventsProcessed = 0
    summaryOutput = []

    while continueIt is True:
        try:
            newResult = res.next(timeout=1)
            if timeouts >= 5:
                print("")
            if newResult is not None:
                if newResult.moduleName is not None:
                    mn = newResult.moduleName
                    mn = mn.replace('___init__', '')
                    mn = mn.replace('_', '.')
                else:
                    mn = ""
                rt = newResult.runTime
                if rt is not None:
                    rt = round(newResult.runTime * 10) / 10.0
                    if not newResult.errors and not newResult.failures:
                        print("\t\t\t\t{0}: {1} tests in {2} secs".format(
                            mn, newResult.testsRun, rt))
                    else:
                        print(
                            "\t\t\t\t{0}: {1} tests, {2} errors {3} failures in {4} secs"
                            .format(mn, newResult.testsRun,
                                    len(newResult.errors),
                                    len(newResult.failures), rt))
            timeouts = 0
            eventsProcessed += 1
            summaryOutput.append(newResult)
        except multiprocessing.TimeoutError:  # @UndefinedVariable
            timeouts += 1
            if timeouts == 5 and eventsProcessed > 0:
                print("Delay in processing, seconds: ", end="")
            elif timeouts == 5:
                print("Starting first modules, should take 5-10 seconds: ",
                      end="")
            if timeouts % 5 == 0:
                print(str(timeouts) + " ", end="")
            if timeouts > maxTimeout and eventsProcessed > 0:
                print("\nToo many delays, giving up...")
                continueIt = False
                printSummary(summaryOutput, timeStart, pathsToRun)
                pool.close()
                exit()
        except StopIteration:
            continueIt = False
            pool.close()
            pool.join()
        except Exception as excp:  # pylint: disable=broad-except
            eventsProcessed += 1
            exceptionLog = ModuleResponse("UntrappedException", None,
                                          "%s" % excp)
            summaryOutput.append(exceptionLog)

    sys.stderr = normalStdError
    printSummary(summaryOutput, timeStart, pathsToRun)
Exemple #10
0
def main(fnAccept=None):
    '''
    `fnAccept` is a list of one or more files to test.  Otherwise runs all.
    '''
    poolSize = common.cpus()

    if pylintRun is None:
        print("make sure that 'sudo pip install pylint' is there. exiting.")
        return

    mg = commonTest.ModuleGather()

    fnPathReject = [
        'demos/',
        'alpha/webapps/server',
        'test/',
        '/ext/',
        #'bar.py',  # used to crash pylint...
        #'repeat.py', # used to hang pylint...
        #'spanner.py', # used to hang pylint...
    ]
    disable = [  # These also need to be changed in MUSIC21BASE/.pylintrc
        'cyclic-import',  # we use these inside functions when there's a deep problem.
        'unnecessary-pass',  # nice, but not really a problem...
        'locally-disabled',  # test for this later, but hopefully will know what 
        # they're doing
        'duplicate-code',  # needs to ignore strings -- keeps getting doctests...
        'arguments-differ',  # someday...
        'abstract-class-instantiated',  # this trips on the fractions.Fraction() class.
        'multiple-imports',  # import os, sys -- fine...
        'fixme',  # known...
        'superfluous-parens',  # nope -- if they make things clearer...
        'too-many-statements',  # someday
        'no-member',  # important, but too many false positives
        'too-many-arguments',  # definitely! but takes too long to get a fix now...
        'too-many-public-methods',  # maybe, look 
        'too-many-branches',  # yes, someday
        'too-many-locals',  # no
        'too-many-lines',  # yes, someday.
        'bad-whitespace',  # maybe later, but "bad" isn't something I necessarily agree with
        'bad-continuation',  # never remove -- this is a good thing many times.
        'too-many-return-statements',  # we'll see
        'unpacking-non-sequence',  # gets it wrong too often.
        'too-many-instance-attributes',  # maybe later
        'too-many-boolean-expressions',  #AbstractDiatonicScale.__eq__ shows how this
        # can be fine...
        'misplaced-comparison-constant',  # sometimes 2 < x is what we want
        'unsubscriptable-object',  # unfortunately, thinks that Streams are unsubscriptable.
        'consider-iterating-dictionary',  # sometimes .keys() is a good test against
        # changing the dictionary size while iterating.
        'invalid-name',  # these are good music21 names; fix the regexp instead...
        'no-self-use',  # maybe later
        'too-few-public-methods',  # never remove or set to 1
        'trailing-whitespace',  # should ignore blank lines with tabs
        'trailing-newlines',  # just because something is easy to detect doesn't make it bad.
        'missing-docstring',  # gets too many well-documented properties
        'star-args',  # no problem with them...
        'protected-access',  # this is an important one, but for now we do a lot of
        # x = copy.deepcopy(self); x._volume = ... which is not a problem...
        'unused-argument',
        'import-self',  # fix is either to get rid of it or move away many tests...
        'redefined-variable-type',  # this would be great! but too much.
        'simplifiable-if-statement',  # NO! NO! NO!
        #  if (x or y and z and q): return True, else: return False,
        #      is a GREAT paradigm -- over "return (x or y and z and q)" and
        #      assuming that it returns a bool...  it's no slower than
        #      the simplification and it's so much clearer.
        'consider-using-enumerate',  # good when i used only once, but
        # x[i] = y[i] is a nice paradigm, even if one can be simplified out.
    ]

    goodnameRx = {
        'argument-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
        'attr-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
        'class-rgx': r'[A-Z_][A-Za-z0-9_]{2,30}$',
        'function-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
        'method-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
        'module-rgx': r'(([a-z_][a-zA-Z0-9_]*)|([A-Z][a-zA-Z0-9]+))$',
        'variable-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
    }

    cmd = [
        '--output-format=parseable',
        r'--dummy-variables-rgx="_$|dummy|unused|i$|j$|junk|counter"',
        '--docstring-min-length=3',
        '--max-args=7',  # should be 5 later, but baby steps
        '--bad-names="foo,shit,f**k,stuff"',  # definitely allow "bar" for barlines
        '--reports=n',
        '--max-branches=20',
        '-j ' + str(poolSize),  # multiprocessing!
        r'--ignore-long-lines="converter\.parse"',  # some tiny notation...
        '--max-line-length=100',  # tada
    ]
    for gn, gnv in goodnameRx.items():
        cmd.append('--' + gn + '="' + gnv + '"')

    #print(cmd)

    for pyLintId in disable:
        cmd.append('--disable=%s' % pyLintId)

    # add entire package
    acceptable = []
    for fp in mg.modulePaths:
        rejectIt = False
        for rejectPath in fnPathReject:
            if rejectPath in fp:
                rejectIt = True
                break
        if rejectIt:
            continue
        if fnAccept:
            rejectIt = True
            for acceptableName in fnAccept:
                if acceptableName in fp:
                    rejectIt = False
                    break
            if rejectIt:
                continue

        acceptable.append(fp)

    cmdFile = cmd + acceptable
    #print(' '.join(cmdFile))
    #print(fp)
    pylintRun(cmdFile, exit=False)
Exemple #11
0
def mainPoolRunner(testGroup=('test',), restoreEnvironmentDefaults=False, leaveOut=1):
    '''
    Run all tests. Group can be test and/or external
    '''    
    normalStdError = sys.stderr
    
    timeStart = time.time()
    poolSize = common.cpus()

    print('Creating %d processes for multiprocessing (omitting %d processors)' % (poolSize, leaveOut))
    

    modGather = commonTest.ModuleGather(useExtended=True)

    maxTimeout = 200
    pathsToRun = modGather.modulePaths # [30:60]


    pool = multiprocessing.Pool(processes=poolSize) # @UndefinedVariable # pylint: disable=not-callable
    
    # imap returns the results as they are completed.  Since the number of files is small,
    # the overhead of returning is outweighed by the positive aspect of getting results immediately
    # unordered says that results can RETURN in any order; not that they'd be pooled out in any
    # order.    
    res = pool.imap_unordered(runOneModuleWithoutImp, 
                              ((modGather, fp) for fp in pathsToRun))

    continueIt = True
    timeouts = 0
    eventsProcessed = 0
    summaryOutput = []
    
    while continueIt is True:
        try:
            newResult = res.next(timeout=1)
            if timeouts >= 5:
                print("")
            if newResult is not None:
                if newResult.moduleName is not None:
                    mn = newResult.moduleName
                    mn = mn.replace('___init__', '')
                    mn = mn.replace('_', '.')
                else:
                    mn = ""
                rt = newResult.runTime
                if rt is not None:
                    rt = round(newResult.runTime * 10)/10.0
                    if not newResult.errors and not newResult.failures:
                        print("\t\t\t\t{0}: {1} tests in {2} secs".format(
                                            mn,
                                            newResult.testsRun,
                                            rt))
                    else:
                        print("\t\t\t\t{0}: {1} tests, {2} errors {3} failures in {4} secs".format(
                                            mn,
                                            newResult.testsRun,
                                            len(newResult.errors),
                                            len(newResult.failures),
                                            rt))
            timeouts = 0
            eventsProcessed += 1
            summaryOutput.append(newResult)
        except multiprocessing.TimeoutError: # @UndefinedVariable
            timeouts += 1
            if timeouts == 5 and eventsProcessed > 0:
                print("Delay in processing, seconds: ", end="")
            elif timeouts == 5:
                print("Starting first modules, should take 5-10 seconds: ", end="")
            if timeouts % 5 == 0:
                print(str(timeouts) + " ", end="")
            if timeouts > maxTimeout and eventsProcessed > 0:
                print("\nToo many delays, giving up...")
                continueIt = False
                printSummary(summaryOutput, timeStart, pathsToRun)
                pool.close()
                exit()
        except StopIteration:
            continueIt = False
            pool.close()    
            pool.join()
        except Exception as excp: # pylint: disable=broad-except
            eventsProcessed += 1
            exceptionLog = ModuleResponse("UntrappedException", None, "%s" % excp)
            summaryOutput.append(exceptionLog)

    sys.stderr = normalStdError
    printSummary(summaryOutput, timeStart, pathsToRun)
Exemple #12
0
def main(fnAccept=None, strict=False):
    '''
    `fnAccept` is a list of one or more files to test.  Otherwise runs all.
    '''
    poolSize = common.cpus()

    if pylintRun is None:
        print("make sure that 'sudo pip3 install pylint' is there. exiting.")
        return

    mg = commonTest.ModuleGather()

    fnPathReject = [
        # 'demos/',
        # 'test/timeGraphs.py',
        '/ext/',
        # 'bar.py',   # used to crash pylint...
        # 'repeat.py',  # used to hang pylint...
        # 'spanner.py',  # used to hang pylint...
    ]

    disable_unless_strict = [
        'too-many-statements',  # someday
        'too-many-arguments',  # definitely! but takes too long to get a fix now...
        'too-many-public-methods',  # maybe, look
        'too-many-branches',  # yes, someday
        'too-many-lines',    # yes, someday.
        'too-many-return-statements',  # we'll see
        'too-many-instance-attributes',  # maybe later
        'inconsistent-return-statements',  # would be nice
        'protected-access',  # this is an important one, but for now we do a lot of
        # x = copy.deepcopy(self); x._volume = ... which is not a problem...
        # also, test suites need to be exempt.
        'keyword-arg-before-vararg',  # a good thing to check for new code, but
        # requires rewriting function signatures in old code

    ]
    disable = [  # These also need to be changed in MUSIC21BASE/.pylintrc
        'arguments-differ',  # -- no -- should be able to add additional arguments so long
        # as initial ones are the same.
        'multiple-imports',  # import os, sys -- fine...
        'redefined-variable-type',  # would be good, but currently
        # lines like: if x: y = note.Note() ; else: y = note.Rest()
        # triggers this, even though y doesn't change.
        'no-else-return',  # these are unnecessary but can help show the flow of thinking.
        'cyclic-import',  # we use these inside functions when there's a deep problem.
        'unnecessary-pass',  # nice, but not really a problem...
        'locally-disabled',  # test for this later, but hopefully will know what
        # they're doing
        'consider-using-get',  # if it can figure out that the default value is something
        # simple, we will turn back on, but until then, no.
        'chained-comparison',  # sometimes simpler that way
        # 'duplicate-code',  # needs to ignore strings -- keeps getting doctests...
        'too-many-ancestors',  # -- 8 is okay.
        'abstract-class-instantiated',  # this trips on the fractions.Fraction() class.
        'fixme',  # known...
        'superfluous-parens',  # nope -- if they make things clearer...
        'no-member',  # important, but too many false positives
        'too-many-locals',   # no
        'bad-whitespace',  # maybe later, but "bad" isn't something I necessarily agree with
        'bad-continuation',  # never remove -- this is a good thing many times.
        'unpacking-non-sequence',  # gets it wrong too often.

        # AbstractDiatonicScale.__eq__ shows how this
        # can be fine...
        'too-many-boolean-expressions',

        'misplaced-comparison-constant',  # sometimes 2 < x is what we want
        'unsubscriptable-object',  # unfortunately, thinks that Streams are unsubscriptable.

        # sometimes .keys() is a good test against
        # changing the dictionary size while iterating.
        'consider-iterating-dictionary',

        'invalid-name',      # these are good music21 names; fix the regexp instead...
        'no-self-use',       # maybe later
        'too-few-public-methods',  # never remove or set to 1

        'trailing-whitespace',  # should ignore blank lines with tabs

        # just because something is easy to detect doesn't make it bad.
        'trailing-newlines',

        'missing-docstring',    # gets too many well-documented properties
        'star-args',  # no problem with them...
        'unused-argument',
        'import-self',  # fix is either to get rid of it or move away many tests...

        'simplifiable-if-statement',  # NO! NO! NO!
        #  if (x or y and z and q): return True, else: return False,
        #      is a GREAT paradigm -- over "return (x or y and z and q)" and
        #      assuming that it returns a bool...  it's no slower than
        #      the simplification and it's so much clearer.
        'consider-using-enumerate',  # good when i used only once, but
        # x[i] = y[i] is a nice paradigm, even if one can be simplified out.
        'not-callable',  # false positives, for instance on x.next()

        'raise-missing-from',  # later.

    ]
    if not strict:
        disable = disable + disable_unless_strict

    goodNameRx = {'argument-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
                  'attr-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
                  'class-rgx': r'[A-Z_][A-Za-z0-9_]{2,30}$',
                  'function-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
                  'method-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
                  'module-rgx': r'(([a-z_][a-zA-Z0-9_]*)|([A-Z][a-zA-Z0-9]+))$',
                  'variable-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
                  }

    maxArgs = 7 if not strict else 5
    maxBranches = 20 if not strict else 10

    cmd = ['--output-format=parseable',
           r'--dummy-variables-rgx="_$|dummy|unused|i$|j$|junk|counter"',
           '--docstring-min-length=3',
           '--ignore-docstrings=yes',
           '--min-similarity-lines=8',
           '--max-args=' + str(maxArgs),  # should be 5 later, but baby steps
           '--bad-names="foo,shit,f**k,stuff"',  # definitely allow "bar" for barlines
           '--reports=n',
           '--max-branches=' + str(maxBranches),
           '-j ' + str(poolSize),  # multiprocessing!
           r'--ignore-long-lines="converter\.parse"',  # some tiny notation...
           '--max-line-length=100',
           ]
    for gn, gnv in goodNameRx.items():
        cmd.append('--' + gn + '="' + gnv + '"')

    for pyLintId in disable:
        cmd.append(f'--disable={pyLintId}')

    # add entire package
    acceptable = []
    for fp in mg.modulePaths:
        rejectIt = False
        for rejectPath in fnPathReject:
            if rejectPath in fp:
                rejectIt = True
                break
        if rejectIt:
            continue
        if fnAccept:
            rejectIt = True
            for acceptableName in fnAccept:
                if acceptableName in fp:
                    rejectIt = False
                    break
            if rejectIt:
                continue

        acceptable.append(fp)

    cmdFile = cmd + acceptable
    if not acceptable:
        print('No matching files were found.')
        return

    # print(fnAccept)
    # print(' '.join(cmdFile))
    # print(fp)

    try:
        # noinspection PyArgumentList,PyCallingNonCallable
        pylintRun(cmdFile, exit=False)
    except TypeError:
        # noinspection PyCallingNonCallable
        pylintRun(cmdFile, do_exit=False)  # renamed in recent versions
Exemple #13
0
def mainPoolRunner(
        testGroup=('test', ), restoreEnvironmentDefaults=False, leaveOut=1):
    '''
    Run all tests. Group can be test and/or external
    '''
    commonTest.testImports()

    normalStdError = sys.stderr

    timeStart = time.time()
    poolSize = common.cpus()

    print(
        f'Creating {poolSize} processes for multiprocessing (omitting {leaveOut} processors)'
    )

    modGather = commonTest.ModuleGather(useExtended=True)

    maxTimeout = 200
    pathsToRun = modGather.modulePaths  # [30:60]

    # pylint: disable=not-callable
    with multiprocessing.Pool(processes=poolSize) as pool:

        # imap returns the results as they are completed.
        # Since the number of files is small, the overhead of returning is
        # outweighed by the positive aspect of getting results immediately
        # unordered says that results can RETURN in any order; not that
        # they'd be pooled out in any order.
        res = pool.imap_unordered(runOneModuleWithoutImp,
                                  ((modGather, fp) for fp in pathsToRun))

        continueIt = True
        timeouts = 0
        eventsProcessed = 0
        summaryOutput = []

        while continueIt is True:
            try:
                newResult = res.next(timeout=1)
                if timeouts >= 5:
                    print('')
                if newResult is not None:
                    if newResult.moduleName is not None:
                        mn = newResult.moduleName
                        mn = mn.replace('___init__', '')
                        mn = mn.replace('_', '.')
                    else:
                        mn = ''
                    rt = newResult.runTime
                    if rt is not None:
                        rt = round(newResult.runTime * 10) / 10.0
                        if not newResult.errors and not newResult.failures:
                            print(
                                f'\t\t\t\t{mn}: {newResult.testsRun} tests in {rt} secs'
                            )
                        else:
                            numErr = len(newResult.errors)
                            numFail = len(newResult.failures)
                            print(
                                f'\t\t\t\t{mn}: {newResult.testsRun} tests, '
                                f'{numErr} errors {numFail} failures in {rt} secs'
                            )
                timeouts = 0
                eventsProcessed += 1
                summaryOutput.append(newResult)
            except multiprocessing.TimeoutError:
                timeouts += 1
                if timeouts == 5 and eventsProcessed > 0:
                    print('Delay in processing, seconds: ', end='')
                elif timeouts == 5:
                    print('Starting first modules, should take 5-10 seconds: ',
                          end='')

                if timeouts % 5 == 0:
                    print(str(timeouts) + ' ', end='', flush=True)
                if timeouts > maxTimeout and eventsProcessed > 0:
                    print('\nToo many delays, giving up...', flush=True)
                    continueIt = False
                    printSummary(summaryOutput, timeStart, pathsToRun)
                    pool.close()
                    sys.exit()
            except StopIteration:
                continueIt = False
                pool.close()
                pool.join()
            except Exception as excp:  # pylint: disable=broad-except
                eventsProcessed += 1
                exceptionLog = ModuleResponse(returnCode='UntrappedException',
                                              moduleName=str(excp))
                summaryOutput.append(exceptionLog)

    sys.stderr = normalStdError
    printSummary(summaryOutput, timeStart, pathsToRun)
Exemple #14
0
def main(fnAccept=None):
    '''
    `fnAccept` is a list of one or more files to test.  Otherwise runs all.
    '''
    poolSize = common.cpus()
    
    if pylintRun is None:
        print("make sure that 'sudo pip install pylint' is there. exiting.")
        return 

    mg = commonTest.ModuleGather()

    fnPathReject = [
                    'demos/',
                    'alpha/webapps/server',
                    'test/',
                    'mxObjects.py',
                    'fromMxObjects.py',
                    'toMxObjects.py',
                    'xmlHandler.py',
                    '/ext/',
                    #'bar.py',  # used to crash pylint...
                    #'repeat.py', # used to hang pylint...
                    #'spanner.py', # used to hang pylint...
                    ]
    disable = [
                'cyclic-import', # we use these inside functions when there's a deep problem.
                'unnecessary-pass', # nice, but not really a problem...
                'locally-disabled', # test for this later, but hopefully will know what 
                            # they're doing

                'duplicate-code', # needs to ignore strings -- keeps getting doctests...

                'arguments-differ', # someday...
                'abstract-class-instantiated', # this trips on the fractions.Fraction() class.
                'fixme', # known...
                'superfluous-parens', # nope -- if they make things clearer...
                'too-many-statements', # someday
                'no-member', # important, but too many false positives
                'too-many-arguments', # definitely! but takes too long to get a fix now...
                'too-many-public-methods', # maybe, look 
                'too-many-branches', # yes, someday
                'too-many-locals',   # no
                'too-many-lines',    # yes, someday.
                'bad-whitespace', # maybe later, but "bad" isn't something I necessarily agree with
                'bad-continuation',  # never remove -- this is a good thing many times.
                'too-many-return-statements', # we'll see
                'unpacking-non-sequence', # gets it wrong too often.
                'too-many-instance-attributes', # maybe later
                
                'invalid-name',      # these are good music21 names; fix the regexp instead...
                'no-self-use',       # maybe later
                'too-few-public-methods', # never remove or set to 1
                'trailing-whitespace',  # should ignore blank lines with tabs
                'missing-docstring',    # gets too many well-documented properties
                'star-args', # no problem with them...
                'protected-access', # this is an important one, but for now we do a lot of
                           # x = copy.deepcopy(self); x._volume = ... which is not a problem...
                'unused-argument',
                'import-self', # fix is either to get rid of it or move away many tests...
               ]

    goodnameRx = {'argument-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
                  'attr-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
                  'class-rgx': r'[A-Z_][A-Za-z0-9_]{2,30}$',
                  'function-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
                  'method-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
                  'module-rgx': r'(([a-z_][a-zA-Z0-9_]*)|([A-Z][a-zA-Z0-9]+))$',
                  'variable-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
                  }

    cmd = ['--output-format=parseable',
           r'--dummy-variables-rgx="_$|dummy|unused|i$|j$|junk|counter"', 
           '--docstring-min-length=3',
           '--max-args=7',  # should be 5 later, but baby steps
           '--bad-names="foo,shit,f**k,stuff"', # definitely allow "bar" for barlines
           '--reports=n',
           '--max-branches=20',
           '-j ' + str(poolSize), # multiprocessing!
           r'--ignore-long-lines="converter\.parse"', # some tiny notation...
           '--max-line-length=100', # tada
           ]
    for gn, gnv in goodnameRx.items():
        cmd.append('--' + gn + '="' + gnv + '"')

    #print(cmd)
    
    for pyLintId in disable:
        cmd.append('--disable=%s' % pyLintId)

    # add entire package
    acceptable = []
    for fp in mg.modulePaths:
        rejectIt = False
        for rejectPath in fnPathReject:
            if rejectPath in fp:
                rejectIt = True
                break
        if rejectIt:
            continue
        if fnAccept:
            rejectIt = True
            for acceptableName in fnAccept:
                if acceptableName in fp:
                    rejectIt = False
                    break
            if rejectIt:
                continue

        acceptable.append(fp)

    cmdFile = cmd + acceptable
    #print(' '.join(cmdFile))
    #print(fp)
    pylintRun(cmdFile, exit=False)
Exemple #15
0
def main(fnAccept=None):
    '''
    `fnAccept` is a list of one or more files to test.  Otherwise runs all.
    '''
    poolSize = common.cpus()

    if pylintRun is None:
        print("make sure that 'sudo pip install pylint' is there. exiting.")
        return

    mg = commonTest.ModuleGather()

    fnPathReject = [
        'demos/',
        'alpha/webapps/server',
        'test/',
        'mxObjects.py',
        'fromMxObjects.py',
        'toMxObjects.py',
        'xmlHandler.py',
        '/ext/',
        #'bar.py',  # used to crash pylint...
        #'repeat.py', # used to hang pylint...
        #'spanner.py', # used to hang pylint...
    ]
    disable = [
        'cyclic-import',  # we use these inside functions when there's a deep problem.
        'unnecessary-pass',  # nice, but not really a problem...
        'locally-disabled',  # test for this later, but hopefully will know what 
        # they're doing
        'duplicate-code',  # needs to ignore strings -- keeps getting doctests...
        'arguments-differ',  # someday...
        'abstract-class-instantiated',  # this trips on the fractions.Fraction() class.
        'fixme',  # known...
        'superfluous-parens',  # nope -- if they make things clearer...
        'too-many-statements',  # someday
        'no-member',  # important, but too many false positives
        'too-many-arguments',  # definitely! but takes too long to get a fix now...
        'too-many-public-methods',  # maybe, look 
        'too-many-branches',  # yes, someday
        'too-many-locals',  # no
        'too-many-lines',  # yes, someday.
        'bad-whitespace',  # maybe later, but "bad" isn't something I necessarily agree with
        'bad-continuation',  # never remove -- this is a good thing many times.
        'too-many-return-statements',  # we'll see
        'unpacking-non-sequence',  # gets it wrong too often.
        'too-many-instance-attributes',  # maybe later
        'invalid-name',  # these are good music21 names; fix the regexp instead...
        'no-self-use',  # maybe later
        'too-few-public-methods',  # never remove or set to 1
        'trailing-whitespace',  # should ignore blank lines with tabs
        'missing-docstring',  # gets too many well-documented properties
        'star-args',  # no problem with them...
        'protected-access',  # this is an important one, but for now we do a lot of
        # x = copy.deepcopy(self); x._volume = ... which is not a problem...
        'unused-argument',
        'import-self',  # fix is either to get rid of it or move away many tests...
    ]

    goodnameRx = {
        'argument-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
        'attr-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
        'class-rgx': r'[A-Z_][A-Za-z0-9_]{2,30}$',
        'function-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
        'method-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
        'module-rgx': r'(([a-z_][a-zA-Z0-9_]*)|([A-Z][a-zA-Z0-9]+))$',
        'variable-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
    }

    cmd = [
        '--output-format=parseable',
        r'--dummy-variables-rgx="_$|dummy|unused|i$|j$|junk|counter"',
        '--docstring-min-length=3',
        '--max-args=7',  # should be 5 later, but baby steps
        '--bad-names="foo,shit,f**k,stuff"',  # definitely allow "bar" for barlines
        '--reports=n',
        '--max-branches=20',
        '-j ' + str(poolSize),  # multiprocessing!
        r'--ignore-long-lines="converter\.parse"',  # some tiny notation...
        '--max-line-length=100',  # tada
    ]
    for gn, gnv in goodnameRx.items():
        cmd.append('--' + gn + '="' + gnv + '"')

    #print(cmd)

    for pyLintId in disable:
        cmd.append('--disable=%s' % pyLintId)

    # add entire package
    acceptable = []
    for fp in mg.modulePaths:
        rejectIt = False
        for rejectPath in fnPathReject:
            if rejectPath in fp:
                rejectIt = True
                break
        if rejectIt:
            continue
        if fnAccept:
            rejectIt = True
            for acceptableName in fnAccept:
                if acceptableName in fp:
                    rejectIt = False
                    break
            if rejectIt:
                continue

        acceptable.append(fp)

    cmdFile = cmd + acceptable
    #print(' '.join(cmdFile))
    #print(fp)
    pylintRun(cmdFile, exit=False)
Exemple #16
0
def main(fnAccept=None, strict=False):
    '''
    `fnAccept` is a list of one or more files to test.  Otherwise runs all.
    '''
    poolSize = common.cpus()

    if pylintRun is None:
        print("make sure that 'sudo pip3 install pylint' is there. exiting.")
        return

    mg = commonTest.ModuleGather()

    fnPathReject = [
                    'demos/',
                    'alpha/webapps/server',
                    'test/timeGraphs.py',
                    '/ext/',
                    #'bar.py',  # used to crash pylint...
                    #'repeat.py', # used to hang pylint...
                    #'spanner.py', # used to hang pylint...
                    ]

    disable_unless_strict = [
                'too-many-statements', # someday
                'too-many-arguments', # definitely! but takes too long to get a fix now...
                'too-many-public-methods', # maybe, look
                'too-many-branches', # yes, someday
                'too-many-lines',    # yes, someday.
                'too-many-return-statements', # we'll see
                'too-many-instance-attributes', # maybe later
                'inconsistent-return-statements', # would be nice
                'protected-access', # this is an important one, but for now we do a lot of
                           # x = copy.deepcopy(self); x._volume = ... which is not a problem...
                           # also, test suites need to be exempt.
                'keyword-arg-before-vararg', # a good thing to check for new code, but
                           # requires rewriting function signatures in old code

    ]
    disable = [  # These also need to be changed in MUSIC21BASE/.pylintrc
                'arguments-differ', # -- no -- should be able to add additional arguments so long
                    # as initial ones are the same.
                'multiple-imports', # import os, sys -- fine...
                'redefined-variable-type', # would be good, but currently
                # lines like: if x: y = note.Note() ; else: y = note.Rest()
                # triggers this, even though y doesn't change.
                'no-else-return', # these are unnessary but can help show the flow of thinking.
                'cyclic-import', # we use these inside functions when there's a deep problem.
                'unnecessary-pass', # nice, but not really a problem...
                'locally-disabled', # test for this later, but hopefully will know what
                            # they're doing
                'consider-using-get', # if it can figure out that the default value is something
                            # simple, we will turn back on, but until then, no.
                'chained-comparison', # sometimes simpler that way
                #'duplicate-code', # needs to ignore strings -- keeps getting doctests...
                'too-many-ancestors', # -- 8 is okay.
                'abstract-class-instantiated', # this trips on the fractions.Fraction() class.
                'fixme', # known...
                'superfluous-parens', # nope -- if they make things clearer...
                'no-member', # important, but too many false positives
                'too-many-locals',   # no
                'bad-whitespace', # maybe later, but "bad" isn't something I necessarily agree with
                'bad-continuation',  # never remove -- this is a good thing many times.
                'unpacking-non-sequence', # gets it wrong too often.
                'too-many-boolean-expressions', #AbstractDiatonicScale.__eq__ shows how this
                    # can be fine...
                'misplaced-comparison-constant', # sometimes 2 < x is what we want
                'unsubscriptable-object', # unfortunately, thinks that Streams are unsubscriptable.
                'consider-iterating-dictionary', # sometimes .keys() is a good test against
                    # changing the dictionary size while iterating.

                'invalid-name',      # these are good music21 names; fix the regexp instead...
                'no-self-use',       # maybe later
                'too-few-public-methods', # never remove or set to 1

                'trailing-whitespace',  # should ignore blank lines with tabs
                'trailing-newlines', # just because something is easy to detect doesn't make it bad.

                'missing-docstring',    # gets too many well-documented properties
                'star-args', # no problem with them...
                'unused-argument',
                'import-self', # fix is either to get rid of it or move away many tests...

                'simplifiable-if-statement', # NO! NO! NO!
                #  if (x or y and z and q): return True, else: return False,
                #      is a GREAT paradigm -- over "return (x or y and z and q)" and
                #      assuming that it returns a bool...  it's no slower than
                #      the simplification and it's so much clearer.
                'consider-using-enumerate', # good when i used only once, but
                    # x[i] = y[i] is a nice paradigm, even if one can be simplified out.
               ]
    if not strict:
        disable = disable + disable_unless_strict

    goodnameRx = {'argument-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
                  'attr-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
                  'class-rgx': r'[A-Z_][A-Za-z0-9_]{2,30}$',
                  'function-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
                  'method-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
                  'module-rgx': r'(([a-z_][a-zA-Z0-9_]*)|([A-Z][a-zA-Z0-9]+))$',
                  'variable-rgx': r'[a-z_][A-Za-z0-9_]{2,30}$',
                  }

    maxArgs = 7 if not strict else 5
    maxBranches = 20 if not strict else 10

    cmd = ['--output-format=parseable',
           r'--dummy-variables-rgx="_$|dummy|unused|i$|j$|junk|counter"',
           '--docstring-min-length=3',
           '--ignore-docstrings=yes',
           '--min-similarity-lines=8',
           '--max-args=' + str(maxArgs),  # should be 5 later, but baby steps
           '--bad-names="foo,shit,f**k,stuff"', # definitely allow "bar" for barlines
           '--reports=n',
           '--max-branches=' + str(maxBranches),
           '-j ' + str(poolSize), # multiprocessing!
           r'--ignore-long-lines="converter\.parse"', # some tiny notation...
           '--max-line-length=100', # tada
           ]
    for gn, gnv in goodnameRx.items():
        cmd.append('--' + gn + '="' + gnv + '"')

    for pyLintId in disable:
        cmd.append('--disable=%s' % pyLintId)

    # add entire package
    acceptable = []
    for fp in mg.modulePaths:
        rejectIt = False
        for rejectPath in fnPathReject:
            if rejectPath in fp:
                rejectIt = True
                break
        if rejectIt:
            continue
        if fnAccept:
            rejectIt = True
            for acceptableName in fnAccept:
                if acceptableName in fp:
                    rejectIt = False
                    break
            if rejectIt:
                continue

        acceptable.append(fp)

    cmdFile = cmd + acceptable
    #print(' '.join(cmdFile))
    #print(fp)
    try:
        pylintRun(cmdFile, exit=False)
    except TypeError:
        pylintRun(cmdFile, do_exit=False) # renamed in recent versions