def main(input_filename: str, output_format: ExportingFormat, output: str,
verbose: int):
configure_logging(verbose)
logger.debug('File {}', input_filename)
originals = tuple()
with open(input_filename, newline='') as f:
reader = csv.reader(f)
# Skip header row
next(reader)
originals = tuple(map(WardCSVRecord.from_csv_row, reader))
logger.debug('Data: {}', originals)
phone_codes = load_phone_area_table()
if output_format == ExportingFormat.FLAT_JSON:
with open(output, 'w') as f:
f.write(
rapidjson.dumps(tuple(a.dict() for a in originals),
indent=2,
ensure_ascii=False))
click.secho(f'Wrote to {output}', file=sys.stderr, fg='green')
elif output_format == ExportingFormat.NESTED_JSON:
provinces = convert_to_nested(originals, phone_codes)
provinces_dicts = tuple(p.dict() for p in provinces.values())
with open(output, 'w') as f:
f.write(
rapidjson.dumps(provinces_dicts, indent=2, ensure_ascii=False))
click.secho(f'Wrote to {output}', file=sys.stderr, fg='green')
elif output_format == ExportingFormat.PYTHON:
folder = Path(
__file__
).parent.parent / 'vietnam_provinces' / 'enums' # type: Path
if folder.exists() and folder.is_file():
click.secho(f'{output} is not a folder.',
file=sys.stderr,
fg='red')
sys.exit(1)
if not folder.exists():
folder.mkdir()
provinces = convert_to_nested(originals, phone_codes)
out_districts = gen_python_district_enums(provinces.values())
out_wards = gen_python_ward_enums(provinces.values())
logger.info('Built AST')
logger.info('Prettify code with Black')
out_districts = black.format_str(out_districts,
mode=black.Mode(
{black.TargetVersion.PY37},
line_length=120))
out_wards = black.format_str(out_wards,
mode=black.Mode(
{black.TargetVersion.PY37},
line_length=120))
file_districts = folder / 'districts.py' # type: Path
file_districts.write_text(out_districts)
click.secho(f'Wrote to {file_districts}', file=sys.stderr, fg='green')
file_wards = folder / 'wards.py' # type: Path
file_wards.write_text(out_wards)
click.secho(f'Wrote to {file_wards}', file=sys.stderr, fg='green')
# Create __init__ file
init_file = folder / '__init__.py' # type: Path
if not init_file.exists():
init_file.touch()
def get_black_mode(src: Path) -> black.Mode:
"""Read the black configuration from pyproject.toml"""
value = black.find_pyproject_toml((str(src),))
if not value:
return black.Mode(line_length=DEFAULT_BLACK_LINE_LENGTH)
config = black.parse_pyproject_toml(value)
return black.Mode(**{
key: value
for key, value in config.items()
})
def to_code(var_name, var_data, line_length):
source = "{} = {}".format(var_name, repr(var_data))
mode = black.Mode(
target_versions={black.TargetVersion.PY27},
line_length=line_length,
)
return black.format_str(source, mode=mode)
def _write_network_stages(self, config_path):
"""
write the networks of the staged network dict into a "networks" folder including
the access dictionary in the init file
:param str config_path:
"""
config_dir = os.path.dirname(config_path)
network_dir = os.path.join(config_dir, "networks")
if not os.path.exists(network_dir):
os.mkdir(network_dir)
init_file = os.path.join(network_dir, "__init__.py")
init_import_code = ""
init_dict_code = "\n\nnetworks_dict = {\n"
for epoch in self.staged_network_dict.keys():
network_path = os.path.join(network_dir, "network_%i.py" % epoch)
pp = pprint.PrettyPrinter(indent=2,
width=150,
**self.pprint_kwargs)
content = "\nnetwork = %s" % pp.pformat(
self.staged_network_dict[epoch])
with open(network_path, "wt", encoding="utf-8") as f:
if self.black_formatting:
content = black.format_str(content, mode=black.Mode())
f.write(content)
init_import_code += "from .network_%i import network as network_%i\n" % (
epoch,
epoch,
)
init_dict_code += " %i: network_%i,\n" % (epoch, epoch)
init_dict_code += "}\n"
self._write_to_file(init_import_code + init_dict_code, init_file)
def test_generate_clients():
print()
path.exists('amazon_sp_api_clients') and shutil.rmtree('amazon_sp_api_clients')
shutil.copytree('amazon_sp_api_static', 'amazon_sp_api_clients')
for src_path in glob('swagger3_apis/*.json'):
module_name = path.split(path.splitext(src_path)[0])[1]
dst_path = path.join('amazon_sp_api_clients', f'{module_name}.py')
with open(src_path, 'r', encoding='utf-8') as f:
data = json.load(f)
content = render(module_name, data)
content = re.sub(r'(?=\n)\s*(?=\n)', '', content)
try:
content = black.format_str(content, mode=black.Mode(line_length=120))
with open(dst_path, 'w', encoding='utf-8') as f:
f.write(content)
except Exception as e:
print(content)
traceback.print_tb(e.__traceback__)
raise
package_name = path.split(path.splitext(src_path)[0])[1]
with open('amazon_sp_api_clients/__init__.py', mode='a', encoding='utf-8') as f:
f.write(f'from . import {package_name}\n')
return
def formatCode(self, sender=None):
import black
# get the code
code = self.code()
# format the code with black
try:
formattedCode = black.format_str(code, mode=black.Mode())
except black.InvalidInput:
return
# set it back in the text view
textView = self.codeView.getNSTextView()
# store current selection by line range
selectedRange = textView.selectedRange()
string = textView.string()
lineRange = string.lineRangeForRange_(selectedRange)
# replace the text
textView.insertText_replacementRange_(formattedCode, (0, string.length()))
# try to reset the selection location back
cursor = (lineRange.location, 0)
try:
textView.setSelectedRange_(cursor)
textView.scrollRangeToVisible_(cursor)
except IndexError:
# fail silently
pass
def normalize_formatting(code: str) -> str:
"""Returns a string of the code with normalized formatting for easier compares"""
code = astor.to_source(ast.parse(code))
try:
return black.format_file_contents(code, fast=True, mode=black.Mode())
except black.NothingChanged:
return code
def test_vyper():
with open("tests/data/Vault.vy", "r") as f:
src = f.read()
assert blackadder.format_str_override(
src,
mode=black.Mode(),
)
def test_patma_hint() -> None:
source, expected = read_data("pattern_matching_simple")
mode = black.Mode(target_versions={black.TargetVersion.PY39})
with pytest.raises(black.parsing.InvalidInput) as exc_info:
assert_format(source, expected, mode, minimum_version=(3, 10))
exc_info.match(black.parsing.PY310_HINT)
def _format(self):
" Format code "
try:
import black
data = request.form["data"]
# Newline fix (remove cr)
data = data.replace("\r", "").rstrip()
mode = black.Mode(
string_normalization=get_plugin_boolean_config(
"string_normalization"),
line_length=get_plugin_int_config("line_length"),
)
data = black.format_str(src_contents=data, mode=mode)
return prepare_api_response(data=data)
except ImportError:
return prepare_api_response(
error_message=
"black dependency is not installed: to install black `pip install black`"
)
except Exception as ex:
logging.error(ex)
return prepare_api_response(
error_message="Error formatting: {message}".format(
message=error_message(ex)))
def make_highlight(language, fontsize, style, inp, line_width) -> None:
"""
Highlight code for keynote.app from clipboard and save result to clipboard.
STYLE Style for code
FONTSIZE Font size to use
LANGUAGE Programming language of source code
INP What is the source of code
LINE-WIDTH python only. Format code to fit width
"""
lexer = get_lexer_by_name(language)
click.echo(f"Using lexer {lexer}")
code = (pyperclip.paste() if inp == "clipboard" else click.edit(
text=pyperclip.paste(), extension=lexer.filenames[0][1:]))
if language == "python":
code = black.format_str(code, mode=black.Mode(line_length=line_width))
click.echo(
f"Got code from clipboard that starts with {click.style(code[:20], fg='blue')}..."
)
res = highlight(
code,
lexer,
get_formatter_by_name("rtf",
style=get_style_by_name(style),
fontsize=fontsize),
)
click.echo("Highlighted")
pyperclip.copy(res)
click.echo("Done!")
def wrap_python_code(code, max_length=100):
# this black currently does not split strings without spaces
while True:
new_code = code
for string_separator in '"', "'":
new_code = re.sub(
'(%s[0-9a-zA-Z\.\-\/\+]{%i,}?%s)' %
(string_separator, max_length + 1, string_separator),
lambda S: S.group(1)[:max_length] + string_separator + ' ' +
string_separator + S.group(1)[max_length:], new_code)
if new_code == code:
break
else:
code = new_code
logger.debug("\033[31mwrapped: %s\033[0m", code)
mode = black.Mode(
target_versions={black.TargetVersion.PY38},
line_length=max_length,
string_normalization=True,
experimental_string_processing=True,
)
# this will also ensure it's valid code
return black.format_str(code, mode=mode)
def test_patma_invalid() -> None:
source, expected = read_data("pattern_matching_invalid")
mode = black.Mode(target_versions={black.TargetVersion.PY310})
with pytest.raises(black.parsing.InvalidInput) as exc_info:
assert_format(source, expected, mode, minimum_version=(3, 10))
exc_info.match("Cannot parse: 10:11")
def check_code_format(notebook):
"""Run black-nb against the given notebook.
Args:
notebook (Path): The notebook filename to run the formatting check against.
Returns:
(bool, black_nb.SubReport): A boolean indicating whether the code would be reformatted
and the corresponding report.
"""
write_back = black.WriteBack.CHECK
mode = black.Mode(
target_versions=TARGET_VERSIONS,
line_length=100,
is_pyi=False,
string_normalization=True,
)
report = format_file_in_place(
src=notebook,
write_back=write_back,
mode=mode,
clear_output=False,
sub_report=SubReport(write_back=write_back),
)
print(str(report))
if (report.change_count > 0) or (report.failure_count > 0):
return True, report
return False, report
def unprocess(self, v: tgtType) -> srcType: # pylint:disable=undefined-variable
import astor # pylint:disable=import-outside-toplevel
source = astor.to_source(v, indent_with="\t")
if self.autopep:
try:
import autopep8 # pylint:disable=import-outside-toplevel
source = autopep8.fix_code(source,
options={
"max_line_length": 100500,
"aggressive": 4,
"experimental": True
})
except ImportError:
warn("Installing `autopep8` may improve results style.")
except BaseException as ex:
warn("Error during autopeping. " + repr(ex))
if self.blacken:
try:
import black # pylint:disable=import-outside-toplevel
source = black.format_str(source,
mode=black.Mode(line_length=100500))
except ImportError:
warn("Installing `black` may improve results style.")
except BaseException as ex:
warn("Error during blackening. " + repr(ex))
return source
def to_py(cfg, prefix: str = "cfg."):
"""
Convert a config object into its equivalent Python code.
Args:
cfg: an omegaconf config object
prefix: root name for the resulting code (default: "cfg.")
Returns:
str of formatted Python code
"""
import black
cfg = OmegaConf.to_container(cfg, resolve=True)
def _to_str(obj, prefix=None, inside_call=False):
if prefix is None:
prefix = []
if isinstance(obj, abc.Mapping) and "_target_" in obj:
# Dict representing a function call
target = _convert_target_to_string(obj.pop("_target_"))
args = []
for k, v in sorted(obj.items()):
args.append(f"{k}={_to_str(v, inside_call=True)}")
args = ", ".join(args)
call = f"{target}({args})"
return "".join(prefix) + call
elif isinstance(obj, abc.Mapping) and not inside_call:
# Dict that is not inside a call is a list of top-level config objects that we
# render as one object per line with dot separated prefixes
key_list = []
for k, v in sorted(obj.items()):
if isinstance(v, abc.Mapping) and "_target_" not in v:
key_list.append(_to_str(v, prefix=prefix + [k + "."]))
else:
key = "".join(prefix) + k
key_list.append(f"{key}={_to_str(v)}")
return "\n".join(key_list)
elif isinstance(obj, abc.Mapping):
# Dict that is inside a call is rendered as a regular dict
return (
"{"
+ ",".join(
f"{repr(k)}: {_to_str(v, inside_call=inside_call)}"
for k, v in sorted(obj.items())
)
+ "}"
)
elif isinstance(obj, list):
return "[" + ",".join(_to_str(x, inside_call=inside_call) for x in obj) + "]"
else:
return repr(obj)
py_str = _to_str(cfg, prefix=[prefix])
try:
return black.format_str(py_str, mode=black.Mode())
except black.InvalidInput:
return py_str
def format(path):
black.format_file_in_place(
src=path,
fast=False,
mode=black.Mode(target_versions={black.TargetVersion.PY38}),
write_back=black.WriteBack.YES,
)
isort.file(path)
def test_remove_with_brackets() -> None:
source, expected = read_data("remove_with_brackets")
assert_format(
source,
expected,
black.Mode(preview=True),
minimum_version=(3, 9),
)
def _code_style(dir: Path) -> None:
for file in dir.rglob('**/*.py'):
black.format_file_in_place(
src=file.absolute(),
fast=False,
mode=black.Mode(line_length=100,
target_versions={black.TargetVersion.PY38}),
write_back=black.WriteBack.YES)
def test_vyper_stable(): # Same as above except fast = False
assert blackadder.reformat_one(
black.Path("data/Vault.vy"),
fast=False,
write_back=black.WriteBack.NO,
mode=black.Mode(),
report=black.Report(check=True),
)
def _write_to_file(self, content, file_path):
"""
write with optional black formatting
:param str content:
:param str config_path:
"""
with open(file_path, "wt", encoding="utf-8") as f:
if self.black_formatting:
content = black.format_str(content, mode=black.Mode())
f.write(content)
def blackify(code):
"""
Applies the black part of our `make style` command to `code`.
"""
has_indent = len(get_indent(code)) > 0
if has_indent:
code = f"class Bla:\n{code}"
mode = black.Mode(target_versions={black.TargetVersion.PY35}, line_length=119, preview=True)
result = black.format_str(code, mode=mode)
result, _ = style_docstrings_in_code(result)
return result[len("class Bla:\n") :] if has_indent else result
def do_black(content, is_pyi):
mode = black.Mode(
target_versions={black.TargetVersion.PY35},
line_length=100,
is_pyi=is_pyi,
string_normalization=True,
experimental_string_processing=False,
)
try:
return black.format_file_contents(content, fast=True, mode=mode)
except black.NothingChanged:
return content
def test_generate_clients():
print()
path.exists('amazon_sp_api_clients') and shutil.rmtree(
'amazon_sp_api_clients')
shutil.copytree('amazon_sp_api_static', 'amazon_sp_api_clients')
with open('amazon_sp_api_clients/__init__.py', mode='r',
encoding='utf-8') as f:
init_lines = f.readlines()
for src_path in map(Path, glob(str(base_dir / 'swagger3_apis/*.json'))):
module_name = src_path.stem
# usually debug a single client, so skip others to speed up
# if 'feed' not in module_name:
# continue
dst_path = path.join('amazon_sp_api_clients', f'{module_name}.py')
with open(src_path, 'r', encoding='utf-8') as f:
data = json.load(f)
context, content = render(module_name, data)
content = re.sub(r'(?=\n)\s*(?=\n)', '', content)
try:
content = black.format_str(content,
mode=black.Mode(line_length=120))
with open(dst_path, 'w', encoding='utf-8') as f:
f.write(content)
except Exception as e:
print(content)
traceback.print_tb(e.__traceback__)
raise
package_name = path.split(path.splitext(src_path)[0])[1]
init_lines.insert(
0, f'from .{package_name} import {context["class_name"]}Client\n')
init_content = ''.join(init_lines)
init_content = black.format_str(init_content,
mode=black.Mode(line_length=120))
with open('amazon_sp_api_clients/__init__.py', mode='w',
encoding='utf-8') as f:
f.write(init_content)
def __init__(self, arguments: Namespace):
self._args = Namespace(
line_length=arguments.black_line_length,
skip_magic_trailing_comma=arguments.
black_skip_magic_trailing_comma,
skip_string_normalization=arguments.
black_skip_string_normalization,
)
self._mode = black.Mode(
line_length=self._args.line_length,
magic_trailing_comma=not self._args.skip_magic_trailing_comma,
string_normalization=not self._args.skip_string_normalization,
)
def __format(self, firstLine: str, output: str):
try:
import black # pylint: disable = import-outside-toplevel
blackConfig = black.parse_pyproject_toml(os.getcwd() + '/pyproject.toml')
blackMode = black.Mode(**blackConfig)
return black.format_str(firstLine + '\n' + output, mode=blackMode)
except ImportError:
if self.__forceEndFileNewLine is True and output[-1:] != '\n':
output += '\n'
return firstLine + '\n' + output
def generate_class(class_name: str, cmd: str, level=0) -> str:
docker_lines = get_help_message(cmd)
nl = level + 4
sections, arguments = parse_help(docker_lines)
cmd_fns = ""
add_imports: Set[str] = set()
if "commands" in sections:
logger.info("Found commands in section {}", cmd)
for cmd_fn, add_import in get_def_commands(sections):
cmd_fns += cmd_fn
add_imports = add_imports.union(add_import)
args: List[str] = _flatten(list(type_arg(arg)) for arg in arguments)
# List of argument that are options only
options = ", ".join([f'"{arg.arg}"' for arg in arguments if arg.is_option])
if options:
options = options + ","
new_line = "\n"
res = indent(
f'''
# DO NOT EDIT: Autogenerated by {__file__}
# for {_version("docker-compose")}
import attr
from typing import Optional, List
from docker_composer.base import DockerBaseRunner
{new_line.join(add_imports)}
@attr.s(auto_attribs=True)
class {class_name}(DockerBaseRunner):
"""
{indent(get_docstring(sections), level=nl)}
"""
{indent(args, level=nl)}
_cmd: str = "{cmd or ""}"
_options: List[str] = [{options}]
{indent(cmd_fns, level=nl)}
''',
level=level,
)
try:
res = isort.code(res,
config=isort.Config(settings_path=project_root()))
except ISortError as exc:
logger.exception(exc)
try:
return black.format_str(res, mode=black.Mode())
except Exception as exc:
logger.exception(exc)
return res
def __init__(
self,
text: str,
font: typing.Union[Font, str] = "Courier",
font_size: Decimal = Decimal(12),
font_color: Color = RGBColor(Decimal(36), Decimal(41), Decimal(46)),
horizontal_alignment: Alignment = Alignment.LEFT,
vertical_alignment: Alignment = Alignment.TOP,
border_top: bool = False,
border_right: bool = False,
border_bottom: bool = False,
border_left: bool = False,
border_color: Color = X11Color("Black"),
border_width: Decimal = Decimal(1),
padding_top: Decimal = Decimal(5),
padding_right: Decimal = Decimal(5),
padding_bottom: Decimal = Decimal(5),
padding_left: Decimal = Decimal(5),
background_color: typing.Optional[Color] = RGBColor(
Decimal(246), Decimal(248), Decimal(250)
),
parent: typing.Optional[LayoutElement] = None,
):
# format string using black
if able_to_import_black:
text = black.format_str(text, mode=black.Mode())
# call super
super().__init__(
text=text,
font=font,
font_size=font_size,
font_color=font_color,
horizontal_alignment=horizontal_alignment,
vertical_alignment=vertical_alignment,
border_top=border_top,
border_right=border_right,
border_bottom=border_bottom,
border_left=border_left,
border_color=border_color,
border_width=border_width,
padding_top=padding_top,
padding_right=padding_right,
padding_bottom=padding_bottom,
padding_left=padding_left,
background_color=background_color,
respect_newlines_in_text=True,
respect_spaces_in_text=True,
parent=parent,
)
def format_code(self, code: str, notebook: bool, **options) -> str:
logging.info(f"Applying {self.label}")
logging.info(f"Options: {options}")
logging.info(f"Line length: {self.line_length}")
black_mode = black.Mode(line_length=self.line_length)
# Note: we cannot use \s for space here, since \s matches any whitespace character, inc. \n
code = re.sub(r" +\n", "\n", code)
# In the future, we might be able to get black mode from options. Not possible at the moment
# see https://github.com/ryantam626/jupyterlab_code_formatter/issues/87
# black_mode = black.Mode(**options)
return reformat_text(code, black_mode)
def recode(self) -> str:
# import was failing on Github actions, therefore here
import black
if self.get_first_line().startswith("```"):
lang = self.get_first_line().replace("```", "").strip()
code = "".join(self.raw_chunk.lines[1:-1])
if lang == "python":
try:
code = black.format_str(code, mode=black.Mode())
except black.InvalidInput as e:
print(e)
return "```" + lang + "\n" + code + "```"
else:
return self.get_content()