def run_pep257(): log.setLevel(logging.DEBUG) opt_parser = get_option_parser() # setup the logger before parsing the config file, so that command line # arguments for debug / verbose will be printed. options, arguments = opt_parser.parse_args() setup_stream_handler(options) # We parse the files before opening the config file, since it changes where # we look for the file. options = get_options(arguments, opt_parser) # Setup the handler again with values from the config file. setup_stream_handler(options) collected = collect(arguments or ['.'], match=re(options.match + '$').match, match_dir=re(options.match_dir + '$').match) log.debug("starting pep257 in debug mode.") Error.explain = options.explain Error.source = options.source collected = list(collected) errors = check(collected, ignore=options.ignore.split(',')) code = 0 count = 0 for error in errors: sys.stderr.write('%s\n' % error) code = 1 count += 1 if options.count: print(count) return code
def check_triple_double_quotes(self, definition, docstring): r'''D300: Use """triple double quotes""". For consistency, always use """triple double quotes""" around docstrings. Use r"""raw triple double quotes""" if you use any backslashes in your docstrings. For Unicode docstrings, use u"""Unicode triple-quoted strings""". Note: Exception to this is made if the docstring contains """ quotes in its body. ''' if docstring: if '"""' in ast.literal_eval(docstring): # Allow ''' quotes if docstring contains """, because # otherwise """ quotes could not be expressed inside # docstring. Not in PEP 257. regex = re(r"[uU]?[rR]?'''[^'].*") else: regex = re(r'[uU]?[rR]?"""[^"].*') if not regex.match(docstring): illegal_matcher = re(r"""[uU]?[rR]?("+|'+).*""") illegal_quotes = illegal_matcher.match(docstring).group(1) return violations.D300(illegal_quotes)
def run_pep257(): log.setLevel(logging.DEBUG) opt_parser = get_option_parser() # setup the logger before parsing the config file, so that command line # arguments for debug / verbose will be printed. options, arguments = opt_parser.parse_args() setup_stream_handlers(options) # We parse the files before opening the config file, since it changes where # we look for the file. options = get_options(arguments, opt_parser) if not validate_options(options): return INVALID_OPTIONS_RETURN_CODE # Setup the handler again with values from the config file. setup_stream_handlers(options) collected = collect(arguments or ['.'], match=re(options.match + '$').match, match_dir=re(options.match_dir + '$').match) log.debug("starting pep257 in debug mode.") Error.explain = options.explain Error.source = options.source collected = list(collected) checked_codes = get_checked_error_codes(options) errors = check(collected, select=checked_codes) code = NO_VIOLATIONS_RETURN_CODE count = 0 for error in errors: sys.stderr.write('%s\n' % error) code = VIOLATIONS_RETURN_CODE count += 1 if options.count: print(count) return code
def main(options, arguments): Error.explain = options.explain Error.source = options.source collected = collect(arguments or ['.'], match=re(options.match + '$').match, match_dir=re(options.match_dir + '$').match) code = 0 for error in check(collected, ignore=options.ignore.split(',')): sys.stderr.write('%s\n' % error) code = 1 return code
def main(options, arguments): if options.debug: log.setLevel(logging.DEBUG) log.debug("starting pep257 in debug mode.") Error.explain = options.explain Error.source = options.source collected = collect(arguments or ['.'], match=re(options.match + '$').match, match_dir=re(options.match_dir + '$').match) code = 0 for error in check(collected, ignore=options.ignore.split(',')): sys.stderr.write('%s\n' % error) code = 1 return code
def _get_ignore_decorators(config): """Return the `ignore_decorators` as None or regex.""" if config.ignore_decorators: # not None and not '' ignore_decorators = re(config.ignore_decorators) else: ignore_decorators = None return ignore_decorators
def _get_leading_words(line): """Return any leading set of words from `line`. For example, if `line` is " Hello world!!!", returns "Hello world". """ result = re("[A-Za-z ]+").match(line.strip()) if result is not None: return result.group()
def is_public(self): # Check if we are a setter/deleter method, and mark as private if so. for decorator in self.decorators: # Given 'foo', match 'foo.bar' but not 'foobar' or 'sfoo' if re(r"^{0}\.".format(self.name)).match(decorator.name): return False name_is_public = not self.name.startswith('_') or is_magic(self.name) return self.parent.is_public and name_is_public
def keys(self, pattern): """ Add abilite to retrieve a list of keys with wildcard pattern. :returns: List keys """ offset = len(self.make_key('')) mask = re(fnmatch.translate(self.make_key(pattern))) return [k[offset:] for k in self._cache.keys() if mask.match(k)]
def parse_options(args=None, config=True, **overrides): # noqa """ Parse options from command line and configuration files. :return argparse.Namespace: """ if args is None: args = [] # Parse args from command string options = PARSER.parse_args(args) options.file_params = dict() options.linter_params = dict() # Override options for k, v in overrides.items(): passed_value = getattr(options, k, _Default()) if isinstance(passed_value, _Default): setattr(options, k, _Default(v)) # Compile options from ini if config: cfg = get_config(str(options.options)) for k, v in cfg.default.items(): LOGGER.info('Find option %s (%s)', k, v) passed_value = getattr(options, k, _Default()) if isinstance(passed_value, _Default): setattr(options, k, _Default(v)) # Parse file related options for name, opts in cfg.sections.items(): if not name.startswith('pylama'): continue if name == cfg.default_section: continue name = name[7:] if name in LINTERS: options.linter_params[name] = dict(opts) continue mask = re(fnmatch.translate(name)) options.file_params[mask] = dict(opts) # Postprocess options opts = dict(options.__dict__.items()) for name, value in opts.items(): if isinstance(value, _Default): setattr(options, name, process_value(name, value.value)) return options
def is_public(self): """Return True iff this method should be considered public.""" # Check if we are a setter/deleter method, and mark as private if so. for decorator in self.decorators: # Given 'foo', match 'foo.bar' but not 'foobar' or 'sfoo' if re(r"^{}\.".format(self.name)).match(decorator.name): return False name_is_public = (not self.name.startswith('_') or self.name in VARIADIC_MAGIC_METHODS or self.is_magic) return self.parent.is_public and name_is_public
def Rue(f,A="",s=0): from random import choice if f in G:f=G[f] else: c=f f=k("",open(f).read()) if f.startswith("import "): a=f.find("\n",8) f=open(f[7:a]).read()+f[a:] a=f.find("\nimport ")+1 while a: b=f.find("\n",a+8) f=f[:a]+open(f[a+7:b]).read()+f[b:] a=f.find("\nimport ",a)+1 f=f.split("\n::=\n") G[c]=f c="" R=[] for lf,C in zip(range(len(f)-1,-1,-1),f): R+=((re(R[0],16).sub,R[1] if len(R) == 2 else R[1:] or "",len(R) == 1) for R in (R.split("::=") for R in c.split("\n") if R)) while 1: while 1: c=C=C.replace("@@",A) for p0,p1,p2 in R: C=p0(choice(p1) if p2 else p1,C,1) if c is not C:break else:break if D:print(" "*s+C) if lf:break a=C.find("}") if a == -1:break while 1: b=C.rfind("{",0,a) c=C.rfind(":",0,b) f=C[c+1:b] b=C[b+1:a] C=C[:c]+(Smod[f](b) if f in Smod else Rue(f,b,s+1))+C[a+1:] a=C.find("}",c) if a == -1:break return C
from html.parser import HTMLParser from re import compile as re from unicodedata import normalize as unicode_normalize from .base import Service from .common import Trait __all__ = ['Duden'] INPUT_MAXIMUM = 100 IGNORE_ARTICLES = ['der', 'das', 'die'] CASE_MATTERS = ['Weg'] SEARCH_FORM = 'http://www.duden.de/suchen/dudenonline' RE_DETAIL = re(r'href="(https?://www\.duden\.de/rechtschreibung/(.+?))"') RE_MP3 = re(r'(Betonung:|Bei der Schreibung) .*?' r'(<em>|»)(.+?)(</em>|«).+?' r'<a .*? href="(https?://www\.duden\.de/_media_/audio/.+?\.mp3)"') HTML_PARSER = HTMLParser() class Duden(Service): """ Provides a Service-compliant implementation for Duden. """ __slots__ = [] NAME = "Duden"
"""Content Negotiation""" from re import compile as re file_type_re = re(r'\.([^\.]+)$') content_types = { 'text': 'text/plain', 'txt': 'text/plain', 'html': 'text/html', 'js': 'text/javascript', 'css': 'text/css', 'png': 'image/png', 'gif': 'image/gif', 'jpg': 'image/jpg', 'py': 'text/plain', } file_types = dict( (value, key) for key, value in content_types.items() ) def get_content_type(file_name = None, file_type = None, content_type = None): if content_type is None: content_type = 'text/plain' if file_type is None and file_name is not None: file_type_search = file_type_re.search(file_name) if file_type_search: file_type_groups = file_type_search.groups() if file_type_groups:
from threading import get_ident as gettid from os import getpid, unlink, F_OK, O_RDONLY, O_WRONLY, O_EXCL, O_CREAT from pathlib import Path from re import compile as re from time import sleep from random import choice from itertools import chain b64bytes = b'+_' b64chars = b64bytes.decode() b64seq = bytes(b64encode(bytes((i << 2,)), b64bytes)[0] for i in range(64)).decode() #b64set = set(b64seq) directory_re = re('[A-Za-z0-9'+b64chars+']{2}') def my_b64encode(b): return b64encode(b, b64bytes).rstrip(b'=').decode() def my_b64decode(s): return b64decode(s + '=' * (-len(s) & 3), b64bytes) class FilesystemListAction(PoolAction): directory = None cursor = None def sync(self): super().sync() return self.cursor
def _get_ignore_decorators(conf): """Return the `ignore_decorators` as None or regex.""" return (re(conf.ignore_decorators) if conf.ignore_decorators else None)
def get_parser(): """ Make command parser for pylama. :return ArgumentParser: """ split_csp_str = lambda s: list( set(i for i in s.strip().split(',') if i)) parser = ArgumentParser(description="Code audit tool for python.") parser.add_argument( "path", nargs='?', default=_Default(CURDIR), help="Path on file or directory.") parser.add_argument( "--verbose", "-v", action='store_true', help="Verbose mode.") parser.add_argument('--version', action='version', version='%(prog)s ' + version) parser.add_argument( "--format", "-f", default=_Default('pep8'), choices=['pep8', 'pylint'], help="Error format.") parser.add_argument( "--select", "-s", default=_Default(''), type=split_csp_str, help="Select errors and warnings. (comma-separated)") parser.add_argument( "--linters", "-l", default=_Default(','.join(DEFAULT_LINTERS)), type=split_csp_str, help=( "Select linters. (comma-separated). Choices are %s." % ','.join(s for s in utils.__all__) )) parser.add_argument( "--ignore", "-i", default=_Default(''), type=split_csp_str, help="Ignore errors and warnings. (comma-separated)") parser.add_argument( "--skip", default=_Default(''), type=lambda s: [re(fnmatch.translate(p)) for p in s.split(',') if p], help="Skip files by masks (comma-separated, Ex. */messages.py)") parser.add_argument( "--complexity", "-c", default=_Default(DEFAULT_COMPLEXITY), type=int, help="Set mccabe complexity.") parser.add_argument("--report", "-r", help="Filename for report.") parser.add_argument( "--hook", action="store_true", help="Install Git (Mercurial) hook.") parser.add_argument( "--async", action="store_true", help="Enable async mode. Usefull for checking a lot of files. " "Dont supported with pylint.") parser.add_argument( "--options", "-o", default=_Default(DEFAULT_INI_PATH), help="Select configuration file. By default is '<CURDIR>/pylama.ini'") return parser
""" Code runnning support. """ import sys from io import StringIO from re import compile as re from .environment import env encoding = re(r'#.*coding[:=]\s*([-\w.]+)') def run_code(): """ Run python code in current buffer. :returns: None """ errors, err = [], '' line1, line2 = env.var('a:line1'), env.var('a:line2') lines = __prepare_lines(line1, line2) if encoding.match(lines[0]): lines.pop(0) if encoding.match(lines[0]): lines.pop(0) elif encoding.match(lines[1]): lines.pop(1) context = dict(__name__='__main__', __file__=env.var('expand("%:p")'), input=env.user_input, raw_input=env.user_input)
def _get_matches(conf): """Return the `match` and `match_dir` functions for `config`.""" match_func = re(conf.match + '$').match match_dir_func = re(conf.match_dir + '$').match return match_func, match_dir_func
def humanize(string): return re(r'(.)([A-Z]+)').sub(r'\1 \2', string).lower()
def next(obj, default=nothing): if default == nothing: return obj.next() else: try: return obj.next() except StopIteration: return default __version__ = '0.5.0-alpha' __all__ = ('check', 'collect') PROJECT_CONFIG = ('setup.cfg', 'tox.ini', '.pep257') humanize = lambda string: re(r'(.)([A-Z]+)').sub(r'\1 \2', string).lower() is_magic = lambda name: name.startswith('__') and name.endswith('__') is_ascii = lambda string: all(ord(char) < 128 for char in string) is_blank = lambda string: not string.strip() leading_space = lambda string: re('\s*').match(string).group() class Value(object): __init__ = lambda self, *args: vars(self).update(zip(self._fields, args)) __hash__ = lambda self: hash(repr(self)) __eq__ = lambda self, other: other and vars(self) == vars(other) def __repr__(self): format_arg = lambda arg: '{}={!r}'.format(arg, getattr(self, arg)) kwargs = ', '.join(format_arg(arg) for arg in self._fields)
from re import compile as re import traceback import botocore def get_object_access_logs(session, since=None): client = session.client('s3') for bucket in get_s3_buckets(session): try: for o in get_gets(client, bucket['Name'], since): yield o except botocore.exceptions.ClientError: pass # Most likely we don't have sufficient permissions to read that bucket. except: traceback.print_exc() log_line_re = re(r'\w+ *([A-Za-z-]+) *\[(\d+/\w+/\d+:\d+:\d+:\d+).{4,10}\].*GET\.OBJECT *([^ ]+) *.*') key_name_re = re(r'(.*/)?\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}-\w+') def parse(lines): for line in lines: if 'GET.OBJECT' not in line: continue match = log_line_re.match(line) if match: t = datetime.strptime(match.group(2), '%d/%b/%Y:%H:%M:%S') yield dict(bucket=match.group(1), time=t, object=match.group(3)) def get_gets(client, bucket_name, since=None): def ls(**kw): return client.list_objects_v2(Bucket=bucket_name, **kw) if since:
from re import compile as re import operator as op from funcparserlib.lexer import make_tokenizer, Token from funcparserlib.parser import (some, a, maybe, finished, skip, many) NUMBER_RE = re('(\d*\.?\d*)') CONVERT = { "bytes": ( ("TB", 1099511627776), ("GB", 1073741824.0), ("MB", 1048576.0), ("KB", 1024.0), ), "bits": ( ("Tb", 1099511627776), ("Gb", 1073741824.0), ("Mb", 1048576.0), ("Kb", 1024.0), ), "bps": ( ("Gbps", 1000000000.0), ("Mbps", 1000000.0), ("Kbps", 1000.0), ), "short": ( ("Tri", 1000000000000.0), ("Bil", 1000000000.0), ("Mil", 1000000.0), ("K", 1000.0), ), "s": ( ("y", 31536000.0), ("M", 2592000.0), ("w", 604800.0), ("d", 86400.0), ("h", 3600.0), ("m", 60.0), ("s", 1.0), ("ms", 0.001), ),
class ConventionChecker: """Checker for PEP 257, NumPy and Google conventions. D10x: Missing docstrings D20x: Whitespace issues D30x: Docstring formatting D40x: Docstring content issues """ NUMPY_SECTION_NAMES = ( 'Short Summary', 'Extended Summary', 'Parameters', 'Returns', 'Yields', 'Other Parameters', 'Raises', 'See Also', 'Notes', 'References', 'Examples', 'Attributes', 'Methods' ) GOOGLE_SECTION_NAMES = ( 'Args', 'Arguments', 'Attention', 'Attributes', 'Caution', 'Danger', 'Error', 'Example', 'Examples', 'Hint', 'Important', 'Keyword Args', 'Keyword Arguments', 'Methods', 'Note', 'Notes', 'Other Parameters', 'Parameters', 'Return', 'Returns', 'Raises', 'References', 'See Also', 'Tip', 'Todo', 'Warning', 'Warnings', 'Warns', 'Yield', 'Yields', ) # Examples that will be matched - # " random: Test" where random will be captured as the param # " random : test" where random will be captured as the param # " random_t (Test) : test " where random_t will be captured as the param GOOGLE_ARGS_REGEX = re( # Matches anything that fulfills all the following conditions: r"^\s*" # Begins with 0 or more whitespace characters r"(\w+)" # Followed by 1 or more unicode chars, numbers or underscores # The above is captured as the first group as this is the paramater name. r"\s*" # Followed by 0 or more whitespace characters r"\(?(.*?)\)?" # Matches patterns contained within round brackets. # The `(.*?)` is the second capturing group which matches any sequence of # characters in a non-greedy way (denoted by the `*?`) r"\s*" # Followed by 0 or more whitespace chars r":" # Followed by a colon ".+" # Followed by 1 or more characters - which is the docstring for the parameter ) def check_source(self, source, filename, ignore_decorators=None): module = parse(StringIO(source), filename) for definition in module: for this_check in self.checks: terminate = False if isinstance(definition, this_check._check_for): skipping_all = (definition.skipped_error_codes == 'all') decorator_skip = ignore_decorators is not None and any( len(ignore_decorators.findall(dec.name)) > 0 for dec in definition.decorators) if not skipping_all and not decorator_skip: error = this_check(self, definition, definition.docstring) else: error = None errors = error if hasattr(error, '__iter__') else [error] for error in errors: if error is not None and error.code not in \ definition.skipped_error_codes: partition = this_check.__doc__.partition('.\n') message, _, explanation = partition error.set_context(explanation=explanation, definition=definition) yield error if this_check._terminal: terminate = True break if terminate: break @property def checks(self): all = [this_check for this_check in vars(type(self)).values() if hasattr(this_check, '_check_for')] return sorted(all, key=lambda this_check: not this_check._terminal) @check_for(Definition, terminal=True) def check_docstring_missing(self, definition, docstring): """D10{0,1,2,3}: Public definitions should have docstrings. All modules should normally have docstrings. [...] all functions and classes exported by a module should also have docstrings. Public methods (including the __init__ constructor) should also have docstrings. Note: Public (exported) definitions are either those with names listed in __all__ variable (if present), or those that do not start with a single underscore. """ if (not docstring and definition.is_public or docstring and is_blank(ast.literal_eval(docstring))): codes = {Module: violations.D100, Class: violations.D101, NestedClass: violations.D106, Method: (lambda: violations.D105() if definition.is_magic else (violations.D107() if definition.is_init else violations.D102())), Function: violations.D103, NestedFunction: violations.D103, Package: violations.D104} return codes[type(definition)]() @check_for(Definition) def check_one_liners(self, definition, docstring): """D200: One-liner docstrings should fit on one line with quotes. The closing quotes are on the same line as the opening quotes. This looks better for one-liners. """ if docstring: lines = ast.literal_eval(docstring).split('\n') if len(lines) > 1: non_empty_lines = sum(1 for l in lines if not is_blank(l)) if non_empty_lines == 1: return violations.D200(len(lines)) @check_for(Function) def check_no_blank_before(self, function, docstring): # def """D20{1,2}: No blank lines allowed around function/method docstring. There's no blank line either before or after the docstring. """ if docstring: before, _, after = function.source.partition(docstring) blanks_before = list(map(is_blank, before.split('\n')[:-1])) blanks_after = list(map(is_blank, after.split('\n')[1:])) blanks_before_count = sum(takewhile(bool, reversed(blanks_before))) blanks_after_count = sum(takewhile(bool, blanks_after)) if blanks_before_count != 0: yield violations.D201(blanks_before_count) if not all(blanks_after) and blanks_after_count != 0: yield violations.D202(blanks_after_count) @check_for(Class) def check_blank_before_after_class(self, class_, docstring): """D20{3,4}: Class docstring should have 1 blank line around them. Insert a blank line before and after all docstrings (one-line or multi-line) that document a class -- generally speaking, the class's methods are separated from each other by a single blank line, and the docstring needs to be offset from the first method by a blank line; for symmetry, put a blank line between the class header and the docstring. """ # NOTE: this gives false-positive in this case # class Foo: # # """Docstring.""" # # # # comment here # def foo(): pass if docstring: before, _, after = class_.source.partition(docstring) blanks_before = list(map(is_blank, before.split('\n')[:-1])) blanks_after = list(map(is_blank, after.split('\n')[1:])) blanks_before_count = sum(takewhile(bool, reversed(blanks_before))) blanks_after_count = sum(takewhile(bool, blanks_after)) if blanks_before_count != 0: yield violations.D211(blanks_before_count) if blanks_before_count != 1: yield violations.D203(blanks_before_count) if not all(blanks_after) and blanks_after_count != 1: yield violations.D204(blanks_after_count) @check_for(Definition) def check_blank_after_summary(self, definition, docstring): """D205: Put one blank line between summary line and description. Multi-line docstrings consist of a summary line just like a one-line docstring, followed by a blank line, followed by a more elaborate description. The summary line may be used by automatic indexing tools; it is important that it fits on one line and is separated from the rest of the docstring by a blank line. """ if docstring: lines = ast.literal_eval(docstring).strip().split('\n') if len(lines) > 1: post_summary_blanks = list(map(is_blank, lines[1:])) blanks_count = sum(takewhile(bool, post_summary_blanks)) if blanks_count != 1: return violations.D205(blanks_count) @staticmethod def _get_docstring_indent(definition, docstring): """Return the indentation of the docstring's opening quotes.""" before_docstring, _, _ = definition.source.partition(docstring) _, _, indent = before_docstring.rpartition('\n') return indent @check_for(Definition) def check_indent(self, definition, docstring): """D20{6,7,8}: The entire docstring should be indented same as code. The entire docstring is indented the same as the quotes at its first line. """ if docstring: indent = self._get_docstring_indent(definition, docstring) lines = docstring.split('\n') if len(lines) > 1: lines = lines[1:] # First line does not need indent. indents = [leading_space(l) for l in lines if not is_blank(l)] if set(' \t') == set(''.join(indents) + indent): yield violations.D206() if (len(indents) > 1 and min(indents[:-1]) > indent or indents[-1] > indent): yield violations.D208() if min(indents) < indent: yield violations.D207() @check_for(Definition) def check_newline_after_last_paragraph(self, definition, docstring): """D209: Put multi-line docstring closing quotes on separate line. Unless the entire docstring fits on a line, place the closing quotes on a line by themselves. """ if docstring: lines = [l for l in ast.literal_eval(docstring).split('\n') if not is_blank(l)] if len(lines) > 1: if docstring.split("\n")[-1].strip() not in ['"""', "'''"]: return violations.D209() @check_for(Definition) def check_surrounding_whitespaces(self, definition, docstring): """D210: No whitespaces allowed surrounding docstring text.""" if docstring: lines = ast.literal_eval(docstring).split('\n') if lines[0].startswith(' ') or \ len(lines) == 1 and lines[0].endswith(' '): return violations.D210() @check_for(Definition) def check_multi_line_summary_start(self, definition, docstring): """D21{2,3}: Multi-line docstring summary style check. A multi-line docstring summary should start either at the first, or separately at the second line of a docstring. """ if docstring: start_triple = [ '"""', "'''", 'u"""', "u'''", 'r"""', "r'''", 'ur"""', "ur'''" ] lines = ast.literal_eval(docstring).split('\n') if len(lines) > 1: first = docstring.split("\n")[0].strip().lower() if first in start_triple: return violations.D212() else: return violations.D213() @check_for(Definition) def check_triple_double_quotes(self, definition, docstring): r'''D300: Use """triple double quotes""". For consistency, always use """triple double quotes""" around docstrings. Use r"""raw triple double quotes""" if you use any backslashes in your docstrings. For Unicode docstrings, use u"""Unicode triple-quoted strings""". Note: Exception to this is made if the docstring contains """ quotes in its body. ''' if docstring: if '"""' in ast.literal_eval(docstring): # Allow ''' quotes if docstring contains """, because # otherwise """ quotes could not be expressed inside # docstring. Not in PEP 257. regex = re(r"[uU]?[rR]?'''[^'].*") else: regex = re(r'[uU]?[rR]?"""[^"].*') if not regex.match(docstring): illegal_matcher = re(r"""[uU]?[rR]?("+|'+).*""") illegal_quotes = illegal_matcher.match(docstring).group(1) return violations.D300(illegal_quotes) @check_for(Definition) def check_backslashes(self, definition, docstring): r'''D301: Use r""" if any backslashes in a docstring. Use r"""raw triple double quotes""" if you use any backslashes (\) in your docstrings. ''' # Just check that docstring is raw, check_triple_double_quotes # ensures the correct quotes. if docstring and '\\' in docstring and not docstring.startswith( ('r', 'ur')): return violations.D301() @check_for(Definition) def check_unicode_docstring(self, definition, docstring): r'''D302: Use u""" for docstrings with Unicode. For Unicode docstrings, use u"""Unicode triple-quoted strings""". ''' if 'unicode_literals' in definition.module.future_imports: return # Just check that docstring is unicode, check_triple_double_quotes # ensures the correct quotes. if docstring and sys.version_info[0] <= 2: if not is_ascii(docstring) and not docstring.startswith( ('u', 'ur')): return violations.D302() @staticmethod def _check_ends_with(docstring, chars, violation): """First line ends with one of `chars`. First line of the docstring should end with one of the characters in `chars`. `chars` supports either a `str` or an `Iterable[str]`. If the condition is evaluated to be false, it raises `violation`. """ if docstring: summary_line = ast.literal_eval(docstring).strip().split('\n')[0] if not summary_line.endswith(chars): return violation(summary_line[-1]) @check_for(Definition) def check_ends_with_period(self, definition, docstring): """D400: First line should end with a period. The [first line of a] docstring is a phrase ending in a period. """ return self._check_ends_with(docstring, '.', violations.D400) @check_for(Definition) def check_ends_with_punctuation(self, definition, docstring): """D415: should end with proper punctuation. The [first line of a] docstring is a phrase ending in a period, question mark, or exclamation point """ return self._check_ends_with(docstring, ('.', '!', '?'), violations.D415) @check_for(Function) def check_imperative_mood(self, function, docstring): # def context """D401: First line should be in imperative mood: 'Do', not 'Does'. [Docstring] prescribes the function or method's effect as a command: ("Do this", "Return that"), not as a description; e.g. don't write "Returns the pathname ...". """ if docstring and not function.is_test: stripped = ast.literal_eval(docstring).strip() if stripped: first_word = stripped.split()[0] check_word = first_word.lower() if check_word in IMPERATIVE_BLACKLIST: return violations.D401b(first_word) try: correct_forms = IMPERATIVE_VERBS.get(stem(check_word)) except UnicodeDecodeError: # This is raised when the docstring contains unicode # characters in the first word, but is not a unicode # string. In which case D302 will be reported. Ignoring. return if correct_forms and check_word not in correct_forms: best = max( correct_forms, key=lambda f: common_prefix_length(check_word, f) ) return violations.D401( best.capitalize(), first_word ) @check_for(Function) def check_no_signature(self, function, docstring): # def context """D402: First line should not be function's or method's "signature". The one-line docstring should NOT be a "signature" reiterating the function/method parameters (which can be obtained by introspection). """ if docstring: first_line = ast.literal_eval(docstring).strip().split('\n')[0] if function.name + '(' in first_line.replace(' ', ''): return violations.D402() @check_for(Function) def check_capitalized(self, function, docstring): """D403: First word of the first line should be properly capitalized. The [first line of a] docstring is a phrase ending in a period. """ if docstring: first_word = ast.literal_eval(docstring).split()[0] if first_word == first_word.upper(): return for char in first_word: if char not in string.ascii_letters and char != "'": return if first_word != first_word.capitalize(): return violations.D403(first_word.capitalize(), first_word) @check_for(Definition) def check_starts_with_this(self, function, docstring): """D404: First word of the docstring should not be `This`. Docstrings should use short, simple language. They should not begin with "This class is [..]" or "This module contains [..]". """ if docstring: first_word = ast.literal_eval(docstring).split()[0] if first_word.lower() == 'this': return violations.D404() @staticmethod def _is_docstring_section(context): """Check if the suspected context is really a section header. Lets have a look at the following example docstring: '''Title. Some part of the docstring that specifies what the function returns. <----- Not a real section name. It has a suffix and the previous line is not empty and does not end with a punctuation sign. This is another line in the docstring. It describes stuff, but we forgot to add a blank line between it and the section name. Parameters <-- A real section name. The previous line ends with ---------- a period, therefore it is in a new grammatical context. param : int examples : list <------- Not a section - previous line doesn't end A list of examples. with punctuation. notes : list <---------- Not a section - there's text after the A list of notes. colon. Notes: <--- Suspected as a context because there's a suffix to the ----- section, but it's a colon so it's probably a mistake. Bla. ''' To make sure this is really a section we check these conditions: * There's no suffix to the section name or it's just a colon AND * The previous line is empty OR it ends with punctuation. If one of the conditions is true, we will consider the line as a section name. """ section_name_suffix = \ context.line.strip().lstrip(context.section_name.strip()).strip() section_suffix_is_only_colon = section_name_suffix == ':' punctuation = [',', ';', '.', '-', '\\', '/', ']', '}', ')'] prev_line_ends_with_punctuation = \ any(context.previous_line.strip().endswith(x) for x in punctuation) this_line_looks_like_a_section_name = \ is_blank(section_name_suffix) or section_suffix_is_only_colon prev_line_looks_like_end_of_paragraph = \ prev_line_ends_with_punctuation or is_blank(context.previous_line) return (this_line_looks_like_a_section_name and prev_line_looks_like_end_of_paragraph) @classmethod def _check_blanks_and_section_underline(cls, section_name, context, indentation): """D4{07,08,09,12,14}, D215: Section underline checks. Check for correct formatting for docstring sections. Checks that: * The line that follows the section name contains dashes (D40{7,8}). * The amount of dashes is equal to the length of the section name (D409). * The section's content does not begin in the line that follows the section header (D412). * The section has no content (D414). * The indentation of the dashed line is equal to the docstring's indentation (D215). """ blank_lines_after_header = 0 for line in context.following_lines: if not is_blank(line): break blank_lines_after_header += 1 else: # There are only blank lines after the header. yield violations.D407(section_name) yield violations.D414(section_name) return non_empty_line = context.following_lines[blank_lines_after_header] dash_line_found = ''.join(set(non_empty_line.strip())) == '-' if not dash_line_found: yield violations.D407(section_name) if blank_lines_after_header > 0: yield violations.D412(section_name) else: if blank_lines_after_header > 0: yield violations.D408(section_name) if non_empty_line.strip() != "-" * len(section_name): yield violations.D409(len(section_name), section_name, len(non_empty_line.strip())) if leading_space(non_empty_line) > indentation: yield violations.D215(section_name) line_after_dashes_index = blank_lines_after_header + 1 # If the line index after the dashes is in range (perhaps we have # a header + underline followed by another section header). if line_after_dashes_index < len(context.following_lines): line_after_dashes = \ context.following_lines[line_after_dashes_index] if is_blank(line_after_dashes): rest_of_lines = \ context.following_lines[line_after_dashes_index:] if not is_blank(''.join(rest_of_lines)): yield violations.D412(section_name) else: yield violations.D414(section_name) else: yield violations.D414(section_name) @classmethod def _check_common_section(cls, docstring, definition, context, valid_section_names): """D4{05,10,11,13}, D214: Section name checks. Check for valid section names. Checks that: * The section name is properly capitalized (D405). * The section is not over-indented (D214). * There's a blank line after the section (D410, D413). * There's a blank line before the section (D411). Also yields all the errors from `_check_blanks_and_section_underline`. """ indentation = cls._get_docstring_indent(definition, docstring) capitalized_section = context.section_name.title() if (context.section_name not in valid_section_names and capitalized_section in valid_section_names): yield violations.D405(capitalized_section, context.section_name) if leading_space(context.line) > indentation: yield violations.D214(capitalized_section) if (not context.following_lines or not is_blank(context.following_lines[-1])): if context.is_last_section: yield violations.D413(capitalized_section) else: yield violations.D410(capitalized_section) if not is_blank(context.previous_line): yield violations.D411(capitalized_section) yield from cls._check_blanks_and_section_underline(capitalized_section, context, indentation) @classmethod def _check_numpy_section(cls, docstring, definition, context): """D406: NumPy-style section name checks. Check for valid section names. Checks that: * The section name has no superfluous suffix to it (D406). Additionally, also yield all violations from `_check_common_section` which are style-agnostic section checks. """ indentation = cls._get_docstring_indent(definition, docstring) capitalized_section = context.section_name.title() yield from cls._check_common_section(docstring, definition, context, cls.NUMPY_SECTION_NAMES) suffix = context.line.strip().lstrip(context.section_name) if suffix: yield violations.D406(capitalized_section, context.line.strip()) @staticmethod def _check_args_section(docstring, definition, context): """D417: `Args` section checks. Check for a valid `Args` or `Argument` section. Checks that: * The section documents all function arguments (D417) except `self` or `cls` if it is a method. """ docstring_args = set() for line in context.following_lines: match = ConventionChecker.GOOGLE_ARGS_REGEX.match(line) if match: docstring_args.add(match.group(1)) function_args = get_function_args(definition.source) # If the method isn't static, then we skip the first # positional argument as it is `cls` or `self` if definition.kind == 'method' and not definition.is_static: function_args = function_args[1:] missing_args = set(function_args) - docstring_args if missing_args: yield violations.D417(", ".join(sorted(missing_args)), definition.name) @classmethod def _check_google_section(cls, docstring, definition, context): """D416: Google-style section name checks. Check for valid section names. Checks that: * The section does not contain any blank line between its name and content (D412). * The section is not empty (D414). * The section name has semicolon as a suffix (D416). Additionally, also yield all violations from `_check_common_section` which are style-agnostic section checks. """ capitalized_section = context.section_name.title() yield from cls._check_common_section(docstring, definition, context, cls.GOOGLE_SECTION_NAMES) suffix = context.line.strip().lstrip(context.section_name) if suffix != ":": yield violations.D416(capitalized_section + ":", context.line.strip()) if capitalized_section in ("Args", "Arguments"): yield from cls._check_args_section(docstring, definition, context) @staticmethod def _get_section_contexts(lines, valid_section_names): """Generate `SectionContext` objects for valid sections. Given a list of `valid_section_names`, generate an `Iterable[SectionContext]` which provides: * Section Name * String value of the previous line * The section line * Following lines till the next section * Line index of the beginning of the section in the docstring * Boolean indicating whether the section is the last section. for each valid section. """ lower_section_names = [s.lower() for s in valid_section_names] def _suspected_as_section(_line): result = get_leading_words(_line.lower()) return result in lower_section_names # Finding our suspects. suspected_section_indices = [i for i, line in enumerate(lines) if _suspected_as_section(line)] SectionContext = namedtuple('SectionContext', ('section_name', 'previous_line', 'line', 'following_lines', 'original_index', 'is_last_section')) # First - create a list of possible contexts. Note that the # `following_lines` member is until the end of the docstring. contexts = (SectionContext(get_leading_words(lines[i].strip()), lines[i - 1], lines[i], lines[i + 1:], i, False) for i in suspected_section_indices) # Now that we have manageable objects - rule out false positives. contexts = (c for c in contexts if ConventionChecker._is_docstring_section(c)) # Now we shall trim the `following lines` field to only reach the # next section name. for a, b in pairwise(contexts, None): end = -1 if b is None else b.original_index yield SectionContext(a.section_name, a.previous_line, a.line, lines[a.original_index + 1:end], a.original_index, b is None) def _check_numpy_sections(self, lines, definition, docstring): """NumPy-style docstring sections checks. Check the general format of a sectioned docstring: '''This is my one-liner. Short Summary ------------- This is my summary. Returns ------- None. ''' Section names appear in `NUMPY_SECTION_NAMES`. Yields all violation from `_check_numpy_section` for each valid Numpy-style section. """ for ctx in self._get_section_contexts(lines, self.NUMPY_SECTION_NAMES): yield from self._check_numpy_section(docstring, definition, ctx) def _check_google_sections(self, lines, definition, docstring): """Google-style docstring section checks. Check the general format of a sectioned docstring: '''This is my one-liner. Note: This is my summary. Returns: None. ''' Section names appear in `GOOGLE_SECTION_NAMES`. Yields all violation from `_check_google_section` for each valid Google-style section. """ for ctx in self._get_section_contexts(lines, self.GOOGLE_SECTION_NAMES): yield from self._check_google_section(docstring, definition, ctx) @check_for(Definition) def check_docstring_sections(self, definition, docstring): """Check for docstring sections.""" if not docstring: return lines = docstring.split("\n") if len(lines) < 2: return yield from self._check_numpy_sections(lines, definition, docstring) yield from self._check_google_sections(lines, definition, docstring)
def _get_matches(config): """Return the `match` and `match_dir` functions for `config`.""" match_func = re(config.match + '$').match match_dir_func = re(config.match_dir + '$').match return match_func, match_dir_func
def leading_space(string): return re('\s*').match(string).group()
def kafka_system_metric(match): return { 'name': 'kafka-' + match.group(2), 'labels': { 'system': match.group(1) } } """ Simple metrics are encoded as strings. Metrics that need a regex to extract labels are encoded as a tuple (regex, parser). """ metrics = { 'org.apache.samza.system.kafka.KafkaSystemProducerMetrics': [ (re('(.*)-(producer-send-failed)'), kafka_system_metric), (re('(.*)-(producer-send-success)'), kafka_system_metric), (re('(.*)-(producer-sends)'), kafka_system_metric), (re('(.*)-(producer-retries)'), kafka_system_metric), (re('(.*)-(flush-ms)'), kafka_system_metric), (re('(.*)-(flush-failed)'), kafka_system_metric), (re('(.*)-(flushes)'), kafka_system_metric), (re('(.*)-(flush-ns)'), kafka_system_metric), 'serialization error', ], 'org.apache.samza.system.kafka.KafkaSystemConsumerMetrics': { (re('(.*)-(\d+)-(bytes-read)'), topic_partition_metric), (re('(.*)-(\d+)-(high-watermark)'), topic_partition_metric), (re('(.*)-(\d+)-(messages-read)'), topic_partition_metric), (re('(.*)-(\d+)-(offset-change)'), topic_partition_metric), (re('(.*)-(\d+)-(messages-behind-high-watermark)'), topic_partition_metric),
class BrowserGenerator(ServiceDialog): """ Provides a dialog for generating many media files to multiple cards from the card browser. """ HELP_USAGE_DESC = "Adding audio to multiple notes" HELP_USAGE_SLUG = 'browser' _RE_WHITESPACE = re(r'\s+') __slots__ = [ '_browser', # reference to the current Anki browser window '_notes', # list of Note objects selected when window opened '_process', # state during processing; see accept() method below ] def __init__(self, browser, *args, **kwargs): """ Sets our title. """ self._browser = browser self._notes = None # set in show() self._process = None # set in accept() super(BrowserGenerator, self).__init__(title="Add TTS Audio to Selected Notes", *args, **kwargs) # UI Construction ######################################################## def _ui_control(self): """ Returns the superclass's text and preview buttons, adding our inputs to control the mass generation process, and then the base class's cancel/OK buttons. """ header = Label("Fields and Handling") header.setFont(self._FONT_HEADER) intro = Note() # see show() for where the text is initialized intro.setObjectName('intro') layout = super(BrowserGenerator, self)._ui_control() layout.addWidget(header) layout.addWidget(intro) layout.addStretch() layout.addLayout(self._ui_control_fields()) layout.addWidget(self._ui_control_handling()) layout.addStretch() layout.addWidget(self._ui_buttons()) return layout def _ui_control_fields(self): """ Returns a grid layout with the source and destination fields. Note that populating the field dropdowns is deferred to the show() event handler because the available fields might change from call to call. """ source_label = Label("Source Field:") source_label.setFont(self._FONT_LABEL) source_dropdown = QtWidgets.QComboBox() source_dropdown.setObjectName('source') dest_label = Label("Destination Field:") dest_label.setFont(self._FONT_LABEL) dest_dropdown = QtWidgets.QComboBox() dest_dropdown.setObjectName('dest') layout = QtWidgets.QGridLayout() layout.addWidget(source_label, 0, 0) layout.addWidget(source_dropdown, 0, 1) layout.addWidget(dest_label, 1, 0) layout.addWidget(dest_dropdown, 1, 1) return layout def _ui_control_handling(self): """ Return the append/overwrite radio buttons and behavior checkbox. """ append = QtWidgets.QRadioButton( "&Append [sound:xxx] Tag onto Destination Field") append.setObjectName('append') append.toggled.connect(self._on_handling_toggled) overwrite = QtWidgets.QRadioButton( "Over&write the Destination Field w/ Media Filename") overwrite.setObjectName('overwrite') overwrite.toggled.connect(self._on_handling_toggled) behavior = Checkbox(object_name='behavior') behavior.stateChanged.connect( lambda status: self._on_behavior_changed(), ) layout = QtWidgets.QVBoxLayout() layout.addWidget(append) layout.addWidget(overwrite) layout.addSpacing(self._SPACING) layout.addWidget(behavior) widget = QtWidgets.QWidget() widget.setLayout(layout) return widget def _ui_buttons(self): """ Adjust title of the OK button. """ buttons = super(BrowserGenerator, self)._ui_buttons() buttons.findChild(QtWidgets.QAbstractButton, 'okay').setText("&Generate") return buttons # Events ################################################################# def show(self, *args, **kwargs): """ Populate the source and destination dropdowns, recall the handling and behavior inputs, and focus the source dropdown. Note that the fields are dumped and repopulated each time, because the list of fields might change between displays of the window. """ self._notes = [ self._browser.mw.col.getNote(note_id) for note_id in self._browser.selectedNotes() ] self.findChild(Note, 'intro').setText( '%d note%s selected. Click "Help" for usage hints.' % (len(self._notes), "s" if len(self._notes) != 1 else "")) fields = sorted( {field for note in self._notes for field in note.keys()}) config = self._addon.config source = self.findChild(QtWidgets.QComboBox, 'source') source.clear() for field in fields: source.addItem(field, field) source.setCurrentIndex( max(source.findData(config['last_mass_source']), 0)) dest = self.findChild(QtWidgets.QComboBox, 'dest') dest.clear() for field in fields: dest.addItem(field, field) dest.setCurrentIndex(max(dest.findData(config['last_mass_dest']), 0)) self.findChild( QtWidgets.QRadioButton, 'append' if config['last_mass_append'] else 'overwrite', ).setChecked(True) self.findChild(Checkbox, 'behavior') \ .setChecked(config['last_mass_behavior']) super(BrowserGenerator, self).show(*args, **kwargs) source.setFocus() def accept(self): """ Check to make sure that we have at least one note, pull the service options, and kick off the processing. """ now = self._get_all() source = now['last_mass_source'] dest = now['last_mass_dest'] append = now['last_mass_append'] behavior = now['last_mass_behavior'] eligible_notes = [ note for note in self._notes if source in note and dest in note ] if not eligible_notes: self._alerts( f"Of the {len(self._notes)} notes selected in the browser, " f"none have both '{source}' and '{dest}' fields." if len(self._notes) > 1 else f"The selected note does not have both " f"'{source}' and '{dest}' fields.", self, ) return self._disable_inputs() svc_id = now['last_service'] options = (None if svc_id.startswith('group:') else now['last_options'][now['last_service']]) self._process = { 'all': now, 'aborted': False, 'progress': _Progress( maximum=len(eligible_notes), on_cancel=self._accept_abort, title="Generating MP3s", addon=self._addon, parent=self, ), 'service': { 'id': svc_id, 'options': options, }, 'fields': { 'source': source, 'dest': dest, }, 'handling': { 'append': append, 'behavior': behavior, }, 'queue': eligible_notes, 'counts': { 'total': len(self._notes), 'elig': len(eligible_notes), 'skip': len(self._notes) - len(eligible_notes), 'done': 0, # all notes processed 'okay': 0, # calls which resulted in a successful MP3 'fail': 0, # calls which resulted in an exception }, 'exceptions': {}, 'throttling': { 'calls': {}, # unthrottled download calls made per service 'sleep': self._addon.config['throttle_sleep'], 'threshold': self._addon.config['throttle_threshold'], }, } self._browser.mw.checkpoint("AwesomeTTS Batch Update") self._process['progress'].show() self._browser.model.beginReset() self._accept_next() def _accept_abort(self): """ Flags that the user has requested that processing stops. """ self._process['aborted'] = True def _accept_next(self): """ Pop the next note off the queue, if not throttled, and process. """ self._accept_update() proc = self._process throttling = proc['throttling'] if proc['aborted'] or not proc['queue']: self._accept_done() return if throttling['calls'] and \ max(throttling['calls'].values()) >= throttling['threshold']: # at least one service needs a break timer = QtCore.QTimer() throttling['timer'] = timer throttling['countdown'] = throttling['sleep'] timer.timeout.connect(self._accept_throttled) timer.setInterval(1000) timer.start() return note = proc['queue'].pop(0) phrase = note[proc['fields']['source']] phrase = self._addon.strip.from_note(phrase) self._accept_update(phrase) def done(): """Count the processed note.""" proc['counts']['done'] += 1 def okay(path): """Count the success and update the note.""" filename = self._browser.mw.col.media.addFile(path) dest = proc['fields']['dest'] note[dest] = self._accept_next_output(note[dest], filename) proc['counts']['okay'] += 1 note.flush() def fail(exception): """Count the failure and the unique message.""" proc['counts']['fail'] += 1 message = exception.message if isinstance(message, str): message = self._RE_WHITESPACE.sub(' ', message).strip() try: proc['exceptions'][message] += 1 except KeyError: proc['exceptions'][message] = 1 def miss(svc_id, count): """Count the cache miss.""" try: throttling['calls'][svc_id] += count except KeyError: throttling['calls'][svc_id] = count callbacks = dict( done=done, okay=okay, fail=fail, miss=miss, # The call to _accept_next() is done via a single-shot QTimer for # a few reasons: keep the UI responsive, avoid a "maximum # recursion depth exceeded" exception if we hit a string of cached # files, and allow time to respond to a "cancel". then=lambda: QtCore.QTimer.singleShot(0, self._accept_next), ) svc_id = proc['service']['id'] want_human = (self._addon.config['filenames_human'] or '{{text}}' if self._addon.config['filenames'] == 'human' else False) if svc_id.startswith('group:'): config = self._addon.config self._addon.router.group(text=phrase, group=config['groups'][svc_id[6:]], presets=config['presets'], callbacks=callbacks, want_human=want_human, note=note) else: self._addon.router(svc_id=svc_id, text=phrase, options=proc['service']['options'], callbacks=callbacks, want_human=want_human, note=note) def _accept_next_output(self, old_value, filename): """ Given a note's old value and our current handling options, returns a new note value using the passed filename. """ proc = self._process if proc['handling']['append']: if proc['handling']['behavior']: return self._addon.strip.sounds.univ(old_value).strip() + \ ' [sound:%s]' % filename elif filename in old_value: return old_value else: return old_value + ' [sound:%s]' % filename else: if proc['handling']['behavior']: return '[sound:%s]' % filename else: return filename def _accept_throttled(self): """ Called for every "timeout" of the timer during a throttling. """ proc = self._process if proc['aborted']: proc['throttling']['timer'].stop() self._accept_done() return proc['throttling']['countdown'] -= 1 self._accept_update() if proc['throttling']['countdown'] <= 0: proc['throttling']['timer'].stop() del proc['throttling']['countdown'] del proc['throttling']['timer'] proc['throttling']['calls'] = {} self._accept_next() def _accept_update(self, detail=None): """ Update the progress bar and message. """ proc = self._process proc['progress'].update( label="finished %d of %d%s\n" "%d successful, %d failed\n" "\n" "%s" % (proc['counts']['done'], proc['counts']['elig'], " (%d skipped)" % proc['counts']['skip'] if proc['counts']['skip'] else "", proc['counts']['okay'], proc['counts']['fail'], "sleeping for %d second%s" % (proc['throttling']['countdown'], "s" if proc['throttling']['countdown'] != 1 else "") if (proc['throttling'] and 'countdown' in proc['throttling']) else " "), value=proc['counts']['done'], detail=detail, ) def _accept_done(self): """ Display statistics and close out the dialog. """ self._browser.model.endReset() proc = self._process proc['progress'].accept() messages = [ "The %d note%s you selected %s been processed. " % ( proc['counts']['total'], "s" if proc['counts']['total'] != 1 else "", "have" if proc['counts']['total'] != 1 else "has", ) if proc['counts']['done'] == proc['counts']['total'] else "%d of the %d note%s you selected %s processed. " % ( proc['counts']['done'], proc['counts']['total'], "s" if proc['counts']['total'] != 1 else "", "were" if proc['counts']['done'] != 1 else "was", ), "%d note%s skipped for not having both the source and " "destination fields. Of those remaining, " % ( proc['counts']['skip'], "s were" if proc['counts']['skip'] != 1 else " was", ) if proc['counts']['skip'] else "During processing, " ] if proc['counts']['fail']: if proc['counts']['okay']: messages.append( "%d note%s successfully updated, but " "%d note%s failed while processing." % ( proc['counts']['okay'], "s were" if proc['counts']['okay'] != 1 else " was", proc['counts']['fail'], "s" if proc['counts']['fail'] != 1 else "", )) else: messages.append("no notes were successfully updated.") messages.append("\n\n") if len(proc['exceptions']) == 1: messages.append("The following problem was encountered:") messages += [ "\n%s (%d time%s)" % (message, count, "s" if count != 1 else "") for message, count in proc['exceptions'].items() ] else: messages.append("The following problems were encountered:") messages += [ "\n- %s (%d time%s)" % (message, count, "s" if count != 1 else "") for message, count in proc['exceptions'].items() ] else: from aqt.utils import showInfo self._alerts = showInfo messages.append("there were no errors.") if proc['aborted']: messages.append("\n\n") messages.append( "You aborted processing. If you want to rollback the changes " "to the notes that were already processed, use the Undo " "AwesomeTTS Batch Update option from the Edit menu.") self._addon.config.update(proc['all']) self._disable_inputs(False) self._notes = None self._process = None super(BrowserGenerator, self).accept() # this alert is done by way of a singleShot() callback to avoid random # crashes on Mac OS X, which happen <5% of the time if called directly QtCore.QTimer.singleShot( 0, lambda: self._alerts("".join(messages), self._browser), ) def _get_all(self): """ Adds support for fields and behavior. """ source, dest, append, behavior = self._get_field_values() # TODO: could be rewritten with {**, **} syntax return dict( list(super(BrowserGenerator, self)._get_all().items()) + [ ('last_mass_append', append), ('last_mass_behavior', behavior), ('last_mass_dest', dest), ('last_mass_source', source), ]) def _get_field_values(self): """ Returns the user's source and destination fields, append state, and handling mode. """ return ( self.findChild(QtWidgets.QComboBox, 'source').currentText(), self.findChild(QtWidgets.QComboBox, 'dest').currentText(), self.findChild(QtWidgets.QRadioButton, 'append').isChecked(), self.findChild(Checkbox, 'behavior').isChecked(), ) def _on_handling_toggled(self): """ Change the text on the behavior checkbox based on the append or overwrite behavior. """ append = self.findChild(QtWidgets.QRadioButton, 'append') behavior = self.findChild(Checkbox, 'behavior') behavior.setText( "Remove Existing [sound:xxx] Tag(s)" if append.isChecked( ) else "Wrap the Filename in [sound:xxx] Tag") behavior.setChecked(True) def _on_behavior_changed(self): """ Display a warning about bare filenames if user selects the override option and disables wrapping the field with a [sound] tag. """ if self.isVisible(): append = self.findChild(QtWidgets.QRadioButton, 'append') behavior = self.findChild(Checkbox, 'behavior') if not (append.isChecked() or behavior.isChecked()): self._alerts( 'Please note that if you use bare filenames, the "Check ' 'Media" feature in Anki will not detect those audio ' "files as in-use, even if you insert the field into your " "templates.", self, )
# -*- coding: UTF-8 -*- import Image from tiles import HEIGHT, WIDTH from locations import locations as get_locations, SCALE from regions import regions2 as get_regions from glob import glob from re import compile as re from os import stat from os.path import isfile from utils import makedirs from scale import scale_command from darken import darken import sys normalized_labels_re = re(r'archive/labels/(.*)\-(..)-normalized\.png') abnormal_labels_re = re(r'archive/labels/(.*)\-([seku][lt]).png') languages = { "s": "Sindarin", "e": "English", "k": "Khuzdul", "u": "Unknown", } alphabets = { "l": "Latin", "t": "Tengwar", } def normalized_labels(): for file_name in glob("archive/labels/*-normalized.png"): parts = normalized_labels_re.match(file_name).groups()
import json import os import sys from re import compile as re from re import M import yaml from tornado import ioloop, log from .alerts import BaseAlert from .handlers import registry from .units import MILLISECOND, TimeUnit LOGGER = log.gen_log COMMENT_RE = re(r'//\s+.*$', M) class Reactor(object): """ Class description. """ defaults = { 'auth_password': None, 'auth_username': None, 'config': None, 'critical_handlers': ['log', 'smtp'], 'debug': False, 'format': 'short', 'graphite_url': 'http://localhost', 'history_size': '1day', 'interval': '10minute',
tuple. If not, returns a tuple with the old path and None. """ old_path = environ.get('PATH_INFO') if old_path: new_path = '/' + '/'.join(component for component in [ component.strip('-.') for component in get_paths.re_excessive_any.sub( '.', get_paths.re_excessive_dashes.sub( '-', get_paths.re_excessive_periods.sub( '.', get_paths.re_filter_characters.sub( '', get_paths.re_filter_encoding.sub( '', old_path))))).lower().split('/') ] if component) if new_path != old_path: return old_path, new_path return old_path, None get_paths.re_excessive_any = re(r'[-.]{2,}') get_paths.re_excessive_dashes = re(r'-{2,}') get_paths.re_excessive_periods = re(r'\.{2,}') get_paths.re_filter_characters = re(r'[^-./A-Za-z0-9]') get_paths.re_filter_encoding = re(r'%[0-9A-Fa-f]{2}')
def leading_space(string): """Return any leading space from `string`.""" return re('\s*').match(string).group()
class Router(object): """ Control migrations. """ filemask = re(r"[\d]{3}_[^\.]+\.py") proxy = Proxy() def __init__(self, migrate_dir, **options): LOGGER.setLevel(options.get('LOGGING', 'WARNING')) if not op.exists(migrate_dir): LOGGER.warn('Migration directory: %s does not exists.', migrate_dir) md(migrate_dir) config = {} if op.exists(op.join(migrate_dir, 'conf.py')): with open(op.join(migrate_dir, 'conf.py')) as f: exec_in(f.read(), config, config) for key in config: if not key.startswith('_'): options[key] = config[key] else: LOGGER.warn( 'Configuration file `conf.py` didnt found in migration directory' ) self.migrate_dir = migrate_dir self.db = options.get('DATABASE') if not isinstance( self.db, (SqliteDatabase, MySQLDatabase, PostgresqlDatabase)) and self.db: self.db = connect(self.db) try: assert self.db self.proxy.initialize(self.db) assert self.proxy.database MigrateHistory.create_table() except (AttributeError, AssertionError): LOGGER.error("Invalid database: %s", self.db) sys.exit(1) except Exception: pass @property def fs_migrations(self): return sorted(''.join(f[:-3]) for f in ls(self.migrate_dir) if self.filemask.match(f)) @property def db_migrations(self): return [mm.name for mm in MigrateHistory.select()] @property def diff(self): db = set(self.db_migrations) return [name for name in self.fs_migrations if name not in db] def run(self, name=None): """ Run migrations. """ LOGGER.info('Start migrations') migrator = Migrator(self.db) if name: return self.run_one(name, migrator) diff = self.diff for name in diff: self.run_one(name, migrator) if not diff: LOGGER.info('Nothing to migrate') def run_one(self, name, migrator): """ Run a migration. """ LOGGER.info('Run "%s"', name) try: with open(op.join(self.migrate_dir, name + '.py')) as f: with self.db.transaction(): code = f.read() scope = {} exec_in(code, scope) migrate = scope.get('migrate', lambda m: None) logging.info('Start migration %s', name) migrate(migrator, self.db) MigrateHistory.create(name=name) logging.info('Migrated %s', name) except Exception as exc: self.db.rollback() LOGGER.error(exc) def create(self, name): """ Create a migration. """ LOGGER.info('Create a migration "%s"', name) num = len(self.fs_migrations) prefix = '{:03}_'.format(num) name = prefix + name + '.py' copy(MIGRATE_TEMPLATE, op.join(self.migrate_dir, name)) LOGGER.info('Migration has created %s', name)
PARSER.add_argument( "--linters", "-l", default=_Default(','.join(DEFAULT_LINTERS)), type=parse_linters, help=( "Select linters. (comma-separated). Choices are %s." % ','.join(s for s in LINTERS.keys()) )) PARSER.add_argument( "--ignore", "-i", default=_Default(''), type=split_csp_str, help="Ignore errors and warnings. (comma-separated)") PARSER.add_argument( "--skip", default=_Default(''), type=lambda s: [re(fnmatch.translate(p)) for p in s.split(',') if p], help="Skip files by masks (comma-separated, Ex. */messages.py)") PARSER.add_argument("--report", "-r", help="Send report to file [REPORT]") PARSER.add_argument( "--hook", action="store_true", help="Install Git (Mercurial) hook.") PARSER.add_argument( "--async", action="store_true", help="Enable async mode. Usefull for checking a lot of files. " "Dont supported with pylint.") PARSER.add_argument( "--options", "-o", default="", help="Select configuration file. By default is '<CURDIR>/pylama.ini'")
""" Code runnning support. """ import sys from re import compile as re from ._compat import StringIO from .environment import env encoding = re(r'#.*coding[:=]\s*([-\w.]+)') def run_code(): """ Run python code in current buffer. :returns: None """ errors, err = [], '' line1, line2 = env.var('a:line1'), env.var('a:line2') lines = __prepare_lines(line1, line2) if encoding.match(lines[0]): lines.pop(0) if encoding.match(lines[0]): lines.pop(0) elif encoding.match(lines[1]): lines.pop(1) context = dict( __name__='__main__', __file__=env.var('expand("%:p")'), input=env.user_input,
def humanize(string): return re(r"(.)([A-Z]+)").sub(r"\1 \2", string).lower()
def __init__(self) -> None: super().__init__(re(r'\.phps?($|/)'))
def get_parser(): """ Make command parser for pylama. :return ArgumentParser: """ split_csp_str = lambda s: list( set(i for i in s.strip().split(',') if i)) parser = ArgumentParser(description="Code audit tool for python.") parser.add_argument( "path", nargs='?', default=_Default(CURDIR), help="Path on file or directory.") parser.add_argument( "--verbose", "-v", action='store_true', help="Verbose mode.") parser.add_argument('--version', action='version', version='%(prog)s ' + version) parser.add_argument( "--format", "-f", default=_Default('pep8'), choices=['pep8', 'pylint'], help="Error format.") parser.add_argument( "--select", "-s", default=_Default(''), type=split_csp_str, help="Select errors and warnings. (comma-separated)") def parse_linters(csp_str): result = list() for name in split_csp_str(csp_str): linter = LINTERS.get(name) if linter: result.append((name, linter)) return result parser.add_argument( "--linters", "-l", default=_Default(','.join(DEFAULT_LINTERS)), type=parse_linters, help=( "Select linters. (comma-separated). Choices are %s." % ','.join(s for s in LINTERS.keys()) )) parser.add_argument( "--ignore", "-i", default=_Default(''), type=split_csp_str, help="Ignore errors and warnings. (comma-separated)") parser.add_argument( "--skip", default=_Default(''), type=lambda s: [re(fnmatch.translate(p)) for p in s.split(',') if p], help="Skip files by masks (comma-separated, Ex. */messages.py)") parser.add_argument( "--complexity", "-c", default=_Default(DEFAULT_COMPLEXITY), type=int, help="Set mccabe complexity.") parser.add_argument("--report", "-r", help="Filename for report.") parser.add_argument( "--hook", action="store_true", help="Install Git (Mercurial) hook.") parser.add_argument( "--async", action="store_true", help="Enable async mode. Usefull for checking a lot of files. " "Dont supported with pylint.") parser.add_argument( "--options", "-o", default=_Default(DEFAULT_INI_PATH), help="Select configuration file. By default is '<CURDIR>/pylama.ini'") return parser
def check_def_documents_parameteres(self, function, docstring): """Definition must document parameters in docstrings. All functions and methods must document each of the parameters in the NumPy format. https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt Parameters ---------- function: str The string text of the function code docstring: str The docstring for the function, or None """ if (not isinstance(function, Method) and not isinstance(function, Function)): return # I use ___ to prefix locals here, because we had a naming conflict # with a function called name and a local variable called name, so I # want to avoid all naming conflicts. ___fail = False # If it does not exist return if not docstring: return # In case of multline function definition with long parameters, we need # to reconstruct the definition from source. function_source = function.source.split("\n") function_def = '' while not function_def.endswith(":"): function_def += function_source.pop(0) function_def = function_def.strip() ___split = function_def.split("(") ___name = ___split[0].split()[1] params = ___split[1].split("):")[0] # We also need to clean up any extra spaces params = re(', +').split(params) params = [entry.split("=")[0] for entry in params] # Emptry string is for when there are no params. IGNORED_PARAMETERS = ["self", "request", "*args", "**kwargs", ""] # Just drop all ignored parameters. This is a lot cleaner. params = list(set(params) - set(IGNORED_PARAMETERS)) # Ignore self if it is the only param if len(params) == 0: return if len(params) > 0: if not re('Parameters\n\s+----------\n').search(docstring): print "[Method %s] Forgot to define parameters section." % ( ___name) ___fail = True for p in params: # Ignore self as a parameter if p in IGNORED_PARAMETERS: continue # name: type # description reg = escape(p) + ': \w+\n\s+\w+' if not re(reg).search(docstring): print "[Method %s] Forgot to document parameter %s" % ( ___name, p) ___fail = True if ___fail: return D410()
""" Code runnning support. """ try: from StringIO import StringIO except ImportError: from io import StringIO import sys from .environment import env from re import compile as re encoding = re(r'#[^\w]+coding:\s+utf.*$') def run_code(): """ Run python code in current buffer. :returns: None """ errors, err = [], '' line1, line2 = env.var('a:line1'), env.var('a:line2') lines = __prepare_lines(line1, line2) for ix in (0, 1): if encoding.match(lines[ix]): lines.pop(ix) context = dict( __name__='__main__',
from re import compile as re from iterkit import all part_finder = re(r'(_+|\d+|[A-Z]+(?![a-z])|[A-Z][a-z]+|[A-Z]+|[a-z]+)') class CaseString(object): def __init__( self, string = None, parts = None, prefix = None, suffix = None ): # UPPER_CASE # lower_case # TitleCase # camelCase # TitleCase1_2 self.string = string if parts is not None: self.parts = parts self.prefix = '' self.suffix = '' else: subparts = part_finder.findall(string)
Simple metrics are encoded as strings. Metrics that need a regex to extract labels are encoded as a tuple (regex, parser). """ metrics = { 'org.apache.samza.system.kafka.KafkaSystemProducerMetrics': [ 'kafka-producer-send-failed', 'kafka-producer-send-success', 'kafka-producer-sends', 'kafka-producer-retries', 'kafka-flush-ms', 'kafka-flush-failed', 'kafka-flushes', 'serialization error', ], 'org.apache.samza.system.kafka.KafkaSystemConsumerMetrics': { (re('(.*)-(\d+)-(bytes-read)'), topic_partition_metric), (re('(.*)-(\d+)-(high-watermark)'), topic_partition_metric), (re('(.*)-(\d+)-(messages-read)'), topic_partition_metric), (re('(.*)-(\d+)-(offset-change)'), topic_partition_metric), (re('(.*)-(\d+)-(messages-behind-high-watermark)'), topic_partition_metric), (re('(.*)-(reconnects)'), system_metric), (re('(.*)-(skipped-fetch-requests)'), system_metric), (re('(.*)-(topic-partitions)'), system_metric), (re('(.*)-SystemStreamPartition \[(.*), (.*), (.*)\]'), kafka_system_stream_partition_metric), 'poll-count', }, 'org.apache.samza.system.SystemProducersMetrics': { 'flushes', 'sends', 'serialization error', (re('(.*)-(flushes)'), source_metric),
from re import compile as re, IGNORECASE, escape QUOTE_RE = re(r"""(['"])((?:'\S|[^'])*)\1""") def gen_underbars(): while True: yield '<em>' yield '</em>' def replace_underbars(text, next): at = text.find('_') if at < 0: return text else: return text[:at] + next() + replace_underbars(text[at+1:], next) def highlighter(text, terms): if not terms: return text def highlight(match): return '<span class="highlight">%s</span>' % match.groups(0)[0] return re(r'(%s)' % '|'.join(escape(term) for term in terms), IGNORECASE).sub(highlight, text) def markup(text, highlight = None): if highlight is None: highlight = [] next_underbar = gen_underbars().next processed = "" while QUOTE_RE.search(text): match = QUOTE_RE.search(text) quote, enquoted = match.groups()
# Postprocess options opts = dict(options.__dict__.items()) for name, value in opts.items(): if isinstance(value, _Default): action = actions.get(name) if action and callable(action.type): value.value = action.type(value.value) setattr(options, name, value.value) # Parse file related options options.file_params = dict() for k, s in config.sections.items(): if k != config.default_section: mask = re(fnmatch.translate(k)) options.file_params[mask] = dict(s) options.file_params[mask]['lint'] = int( options.file_params[mask].get('lint', 1) ) return options def setup_logger(options): """ Setup logger with options. """ LOGGER.setLevel(logging.INFO if options.verbose else logging.WARN) if options.report: LOGGER.removeHandler(STREAM) LOGGER.addHandler(logging.FileHandler(options.report, mode='w'))
def highlighter(text, terms): if not terms: return text def highlight(match): return '<span class="highlight">%s</span>' % match.groups(0)[0] return re(r'(%s)' % '|'.join(escape(term) for term in terms), IGNORECASE).sub(highlight, text)
import os from re import compile as re, M import json from tornado import ioloop, log from .alerts import BaseAlert from .utils import parse_interval from .handlers import registry LOGGER = log.gen_log COMMENT_RE = re('//\s+.*$', M) class Reactor(object): """ Class description. """ defaults = { 'auth_password': None, 'auth_username': None, 'config': 'config.json', 'critical_handlers': ['log', 'smtp'], 'debug': False, 'format': 'short', 'graphite_url': 'http://localhost', 'history_size': '1day', 'interval': '10minute', 'logging': 'info',
class Router(object): """Control migrations.""" filemask = re(r"[\d]{3}_[^\.]+\.py") def __init__(self, plugin): """Initialize the router.""" self.app = plugin.app self.database = plugin.database self.migrate_dir = plugin.cfg.migrations_path @cached_property def model(self): """Ensure that migrations has prepared to run.""" # Initialize MigrationHistory model MigrateHistory._meta.database = self.app.plugins.peewee.database try: MigrateHistory.create_table() except pw.DatabaseError: self.database.rollback() return MigrateHistory @property def fs_migrations(self): """Scan migrations in file system.""" if not op.exists(self.migrate_dir): self.app.logger.warn('Migration directory: %s does not exists.', self.migrate_dir) md(self.migrate_dir) return sorted(''.join(f[:-3]) for f in ls(self.migrate_dir) if self.filemask.match(f)) @property def db_migrations(self): """Scan migrations in database.""" return [mm.name for mm in self.model.select()] @property def diff(self): """Calculate difference between fs and db.""" dbms = set(self.db_migrations) return [name for name in self.fs_migrations if name not in dbms] def create(self, name='auto'): """Create a migration.""" self.app.logger.info('Create a migration "%s"', name) num = len(self.fs_migrations) prefix = '{:03}_'.format(num) name = prefix + name + '.py' path = copy(MIGRATE_TEMPLATE, op.join(self.migrate_dir, name)) self.app.logger.info('Migration has created %s', path) return path def run(self, name=None): """Run migrations.""" self.app.logger.info('Start migrations') migrator = Migrator(self.database, app=self.app) diff = self.diff if not diff: self.app.logger.info('Nothing to migrate') return None for mname in self.fs_migrations: self.run_one(mname, migrator, mname not in diff) if name and name == mname: break def run_one(self, name, migrator, fake=True): """Run a migration.""" if not fake: self.app.logger.info('Run "%s"', name) try: with open(op.join(self.migrate_dir, name + '.py')) as f: code = f.read() scope = {} exec_in(code, scope) migrate = scope.get('migrate', lambda m: None) migrate(migrator, self.database, app=self.app) if fake: migrator.clean() return migrator self.app.logger.info('Start migration %s', name) with self.database.transaction(): migrator.run() self.model.create(name=name) self.app.logger.info('Migrated %s', name) except Exception as exc: self.database.rollback() self.app.logger.exception(exc) raise