def get_records(
self, workunit: WorkUnit
) -> Iterable[RecordEnvelope[
Union[MetadataChangeEvent, MetadataChangeProposal,
MetadataChangeProposalWrapper, UsageAggregationClass, ]]]:
if isinstance(workunit, MetadataWorkUnit):
if isinstance(
workunit.metadata,
(
MetadataChangeEvent,
MetadataChangeProposal,
MetadataChangeProposalWrapper,
),
):
workunit.metadata.systemMetadata = SystemMetadata(
lastObserved=get_sys_time(), runId=self.ctx.run_id)
if (isinstance(workunit.metadata, MetadataChangeEvent) and len(
workunit.metadata.proposedSnapshot.aspects) == 0):
raise AttributeError(
"every mce must have at least one aspect")
if not workunit.metadata.validate():
invalid_mce = str(workunit.metadata)
if black is not None:
invalid_mce = black.format_str(invalid_mce,
mode=black.FileMode())
raise ValueError(
f"source produced an invalid metadata work unit: {invalid_mce}"
)
yield RecordEnvelope(
workunit.metadata,
{
"workunit_id": workunit.id,
},
)
elif isinstance(workunit, UsageStatsWorkUnit):
if not workunit.usageStats.validate():
invalid_usage_stats = str(workunit.usageStats)
if black is not None:
invalid_usage_stats = black.format_str(
invalid_usage_stats, mode=black.FileMode())
raise ValueError(
f"source produced an invalid usage stat: {invalid_usage_stats}"
)
yield RecordEnvelope(
workunit.usageStats,
{
"workunit_id": workunit.id,
},
)
else:
raise ValueError(f"unknown WorkUnit type {type(workunit)}")
def _test_transformation(self, before, after):
"""Assert that the ``before`` string transformed to the ``after`` string"""
tree = ast.parse(before)
tree = ScriptTransformer().visit(tree)
source = black.format_str(astor.to_source(tree),
mode=black.FileMode(line_length=88))
self.assertEqual(
source, black.format_str(after,
mode=black.FileMode(line_length=88)))
def main(input_filename: str, output_format: ExportingFormat, output: str,
verbose: int):
configure_logging(verbose)
logger.debug('File {}', input_filename)
originals = tuple()
with open(input_filename, newline='') as f:
reader = csv.reader(f)
# Skip header row
next(reader)
originals = tuple(map(WardCSVRecord.from_csv_row, reader))
logger.debug('Data: {}', originals)
phone_codes = load_phone_area_table()
if output_format == ExportingFormat.FLAT_JSON:
with open(output, 'w') as f:
f.write(
rapidjson.dumps(tuple(a.dict() for a in originals),
indent=2,
ensure_ascii=False))
click.secho(f'Wrote to {output}', file=sys.stderr, fg='green')
elif output_format == ExportingFormat.NESTED_JSON:
provinces = convert_to_nested(originals, phone_codes)
provinces_dicts = tuple(p.dict() for p in provinces.values())
with open(output, 'w') as f:
f.write(
rapidjson.dumps(provinces_dicts, indent=2, ensure_ascii=False))
click.secho(f'Wrote to {output}', file=sys.stderr, fg='green')
elif output_format == ExportingFormat.PYTHON:
folder = Path(
__file__
).parent.parent / 'vietnam_provinces' / 'enums' # type: Path
if folder.exists() and folder.is_file():
click.secho(f'{output} is not a folder.',
file=sys.stderr,
fg='red')
sys.exit(1)
if not folder.exists():
folder.mkdir()
provinces = convert_to_nested(originals, phone_codes)
out_districts = gen_python_district_enums(provinces.values())
out_wards = gen_python_ward_enums(provinces.values())
logger.info('Built AST')
logger.info('Prettify code with Black')
out_districts = black.format_str(out_districts,
mode=black.FileMode(line_length=120))
out_wards = black.format_str(out_wards,
mode=black.FileMode(line_length=120))
file_districts = folder / 'districts.py' # type: Path
file_districts.write_text(out_districts)
click.secho(f'Wrote to {file_districts}', file=sys.stderr, fg='green')
file_wards = folder / 'wards.py' # type: Path
file_wards.write_text(out_wards)
click.secho(f'Wrote to {file_wards}', file=sys.stderr, fg='green')
# Create __init__ file
init_file = folder / '__init__.py' # type: Path
if not init_file.exists():
init_file.touch()
def format_code(self, code: str, **options) -> str:
import black
code = re.sub("^%", "#%#", code, flags=re.M)
if black.__version__ >= '19.3b0':
code = black.format_str(code, mode=black.FileMode(**options))[:-1]
else:
code = black.format_str(code, **options)[:-1]
code = re.sub("^#%#", "%", code, flags=re.M)
return code
def compare_diff(mappings):
"""
Generates an HTML object with a diff view of two mappings
"""
if len(mappings) != 2:
return None
m1, m2 = mappings
s1 = black.format_str(str(m1), mode=_fm).splitlines()
s2 = black.format_str(str(m2), mode=_fm).splitlines()
return HTML(_htmldiff.make_file(s1, s2))
def black(self, line, cell):
"""Magic command to format the IPython cell."""
args = magic_arguments.parse_argstring(self.black, line)
line_length = args.line_length
if cell:
try:
from black import FileMode
mode = FileMode(line_length=line_length)
formatted = format_str(src_contents=cell, mode=mode)
except TypeError:
formatted = format_str(src_contents=cell,
line_length=line_length)
if formatted and formatted[-1] == "\n":
formatted = formatted[:-1]
self.shell.set_next_input(formatted, replace=True)
def check(config: GeneratorConfig, output_dir: str, output: OrderedObj[str]):
for path, contents in output:
if not path.startswith(config.codegen_root + "/"):
continue
full_path = os.path.join(output_dir, path)
if contents:
if not os.path.isfile(full_path):
raise ValueError(f"Could not find code generated file at path {path}")
existing_contents = open(full_path, "r", encoding="utf8").read()
contents_formatted = black.format_str(contents, mode=black.FileMode())
existing_contents_formatted = black.format_str(existing_contents, mode=black.FileMode())
if contents_formatted != existing_contents_formatted:
raise ValueError(f"Codegen output is stale for path {path}")
def formatSource(sourceText):
formattedSource = sourceText
try:
formattedSource = format_str(sourceText, 120)
except Exception as e:
print(e)
return formattedSource
def _format(self):
" Format code "
try:
import black
data = request.form["data"]
# Newline fix (remove cr)
data = data.replace("\r", "").rstrip()
mode = black.Mode(
string_normalization=get_plugin_boolean_config(
"string_normalization"),
line_length=get_plugin_int_config("line_length"),
)
data = black.format_str(src_contents=data, mode=mode)
return prepare_api_response(data=data)
except ImportError:
return prepare_api_response(
error_message=
"black dependency is not installed: to install black `pip install black`"
)
except Exception as ex:
logging.error(ex)
return prepare_api_response(
error_message="Error formatting: {message}".format(
message=error_message(ex)))
def unprocess(self, v: tgtType) -> srcType: # pylint:disable=undefined-variable
import astor # pylint:disable=import-outside-toplevel
source = astor.to_source(v, indent_with="\t")
if self.autopep:
try:
import autopep8 # pylint:disable=import-outside-toplevel
source = autopep8.fix_code(source,
options={
"max_line_length": 100500,
"aggressive": 4,
"experimental": True
})
except ImportError:
warn("Installing `autopep8` may improve results style.")
except BaseException as ex:
warn("Error during autopeping. " + repr(ex))
if self.blacken:
try:
import black # pylint:disable=import-outside-toplevel
source = black.format_str(source,
mode=black.Mode(line_length=100500))
except ImportError:
warn("Installing `black` may improve results style.")
except BaseException as ex:
warn("Error during blackening. " + repr(ex))
return source
def test_generate_clients():
print()
path.exists('amazon_sp_api_clients') and shutil.rmtree('amazon_sp_api_clients')
shutil.copytree('amazon_sp_api_static', 'amazon_sp_api_clients')
for src_path in glob('swagger3_apis/*.json'):
module_name = path.split(path.splitext(src_path)[0])[1]
dst_path = path.join('amazon_sp_api_clients', f'{module_name}.py')
with open(src_path, 'r', encoding='utf-8') as f:
data = json.load(f)
content = render(module_name, data)
content = re.sub(r'(?=\n)\s*(?=\n)', '', content)
try:
content = black.format_str(content, mode=black.Mode(line_length=120))
with open(dst_path, 'w', encoding='utf-8') as f:
f.write(content)
except Exception as e:
print(content)
traceback.print_tb(e.__traceback__)
raise
package_name = path.split(path.splitext(src_path)[0])[1]
with open('amazon_sp_api_clients/__init__.py', mode='a', encoding='utf-8') as f:
f.write(f'from . import {package_name}\n')
return
async def generate(schema_name: str = "public"):
"""
Automatically generates Piccolo Table classes by introspecting the
database. Please check the generated code in case there are errors.
"""
output_schema = await get_output_schema(schema_name=schema_name)
output = output_schema.imports + [
i._table_str(excluded_params=["choices"]) for i in output_schema.tables
]
if output_schema.warnings:
warning_str = "\n".join(output_schema.warnings)
output.append('"""')
output.append(
"WARNING: Unrecognised column types, added `Column` as a "
"placeholder:")
output.append(warning_str)
output.append('"""')
nicely_formatted = black.format_str("\n".join(output),
mode=black.FileMode(line_length=79))
print(nicely_formatted)
def make_ex_service(name, ex_service_data, application):
template = env_component.get_template("exservice")
structure = []
for ex_service, ex_service_meta in ex_service_data.items():
ex_service = ex_service.replace("api", "service")
ex_service_name = uppercamelcase(ex_service)
contents = make_ex_service_content(ex_service_name, ex_service_meta)
ex_api_json = template.render(
name=ex_service_name,
ex_service_meta=ex_service_meta,
snakecase=snakecase,
application=application,
contents=contents,
)
ex_api_json = clean_json(ex_api_json)
api_class = make_class(json.loads(ex_api_json))
structure.append(
{
"type": "DATA",
"name": f"{snakecase(ex_service_name)}.py",
"data": black.format_str(api_class, mode=black_mode),
}
)
return {"type": "PACKAGE", "name": snakecase(name), "sub": structure}
def repr_method(self):
init_signature = inspect.signature(self.__init__)
exclude_param_names = set(['self'])
if _repr_depth[0] > MAX_REPR_DEPTH:
return f"{self.__class__.__name__}(...)"
_repr_depth[0] += 1
parameter_names = [
name for name in init_signature.parameters
if name not in exclude_param_names
]
parameter_values = [
getattr(self, param, None) for param in parameter_names
]
if hasattr(self, '__nice_repr_hook__'):
self.__nice_repr_hook__(parameter_names, parameter_values)
args = ", ".join(
f"{name}={repr(value)}"
for name, value in zip(parameter_names, parameter_values)
if value is not None)
fr = f"{self.__class__.__name__}({args})"
_repr_depth[0] -= 1
try:
import black
return black.format_str(fr, mode=black.FileMode()).strip()
except:
return fr
def new_app(app_name: str, root: str = "."):
print(f"Creating {app_name} app ...")
app_root = os.path.join(root, app_name)
if os.path.exists(app_root):
sys.exit("Folder already exists - exiting.")
os.mkdir(app_root)
with open(os.path.join(app_root, "__init__.py"), "w"):
pass
templates: t.Dict[str, t.Any] = {
"piccolo_app.py": {"app_name": app_name},
"tables.py": {},
}
for filename, context in templates.items():
with open(os.path.join(app_root, filename), "w") as f:
template = JINJA_ENV.get_template(filename + ".jinja")
file_contents = template.render(**context)
file_contents = black.format_str(
file_contents, mode=black.FileMode(line_length=80)
)
f.write(file_contents)
migrations_folder_path = os.path.join(app_root, "piccolo_migrations")
os.mkdir(migrations_folder_path)
with open(os.path.join(migrations_folder_path, "__init__.py"), "w"):
pass
def get_signature(name, thing):
"""
Get the signature for a function or class, formatted nicely if possible.
Parameters
----------
name : str
Name of the thing, used as the first part of the signature
thing : class or function
Thing to get the signature of
"""
if inspect.ismodule(thing):
return ""
if isinstance(thing, property):
func_sig = name
else:
try:
sig = inspect.signature(thing)
except TypeError:
sig = inspect.signature(thing.fget)
except ValueError:
return ""
func_sig = f"{name}{sig}"
try:
mode = black.FileMode(line_length=80)
func_sig = black.format_str(func_sig, mode=mode).strip()
except (ValueError, TypeError):
pass
return f"```python\n{func_sig}\n```\n"
def code(objeto):
"""
Genera un _code fencing_ en Markdown y ordena su visualización
Se espera que el *objeto* tenga asociado un archivo con su
código fuente. El resultado puede diferir del código literal ya
que se analizará el árbol sintáctico abstracto, se eliminarán
los _docstring_ y se reconstruirá el código. Se creará un salida
del tipo :class:`IPython.core.display.Markdown`
:param objeto: entidad de la que se desea el código fuente
:type objeto: object
"""
tree = parse(getsource(objeto))
for node in walk(tree):
try:
node.body = list(
filter(
lambda x: not isinstance(x, Expr) or not isinstance(
x.value, Str),
node.body,
))
except BaseException:
pass
display(
Markdown("```python\n" +
format_str(unparse(tree).strip(), mode=FileMode()) + "\n```"))
def format_python_source(
source: str,
*,
line_length: int = 100,
target_versions: Optional[Set[str]] = None,
string_normalization: bool = True,
) -> str:
"""Format Python source code using black formatter."""
target_versions = target_versions or {
f"{sys.version_info.major}{sys.version_info.minor}"
}
target_versions = set(black.TargetVersion[f"PY{v.replace('.', '')}"]
for v in target_versions)
formatted_source = black.format_str(
source,
mode=black.FileMode(
line_length=line_length,
target_versions=target_versions,
string_normalization=string_normalization,
),
)
assert isinstance(formatted_source, str)
return formatted_source
def _operator_jsn_to_string(jsn: JSON_TYPE,
show_imports: bool,
combinators: bool,
astype=str) -> str:
gen = _CodeGenState(_collect_names(jsn), combinators, astype)
expr = _operator_jsn_to_string_rec('pipeline', jsn, gen)
if expr != 'pipeline':
gen.assigns.append(f'pipeline = {expr}')
if show_imports and len(gen.imports) > 0:
if combinators:
gen.imports.append('import lale')
imports_set: Set[str] = set()
imports_list: List[str] = []
for imp in gen.imports:
if imp not in imports_set:
imports_set |= {imp}
imports_list.append(imp)
result = '\n'.join(imports_list)
if combinators:
result += '\nlale.wrap_imported_operators()'
result += '\n'
result += '\n'.join(gen.assigns)
else:
result = '\n'.join(gen.assigns)
formatted = black.format_str(result, mode=_black78).rstrip()
return formatted
def stringify_translated_tool(tool):
try:
import black
return black.format_str(tool, mode=black.FileMode(line_length=82))
except ImportError:
return tool
def test_idempotent_any_syntatically_valid_python(
src_contents: str, mode: black.FileMode) -> None:
# Before starting, let's confirm that the input string is valid Python:
compile(src_contents, "<string>",
"exec") # else the bug is in hypothesmith
# Then format the code...
try:
dst_contents = black.format_str(src_contents, mode=mode)
except black.InvalidInput:
# This is a bug - if it's valid Python code, as above, Black should be
# able to cope with it. See issues #970, #1012, #1358, and #1557.
# TODO: remove this try-except block when issues are resolved.
return
except TokenError as e:
if ( # Special-case logic for backslashes followed by newlines or end-of-input
e.args[0] == "EOF in multi-line statement"
and re.search(r"\\($|\r?\n)", src_contents) is not None):
# This is a bug - if it's valid Python code, as above, Black should be
# able to cope with it. See issue #1012.
# TODO: remove this block when the issue is resolved.
return
raise
# And check that we got equivalent and stable output.
black.assert_equivalent(src_contents, dst_contents)
black.assert_stable(src_contents, dst_contents, mode=mode)
def black_reformat_handler(text_before_cursor):
import black
formatted_text = black.format_str(text_before_cursor,
mode=black.FileMode())
if not text_before_cursor.endswith('\n') and formatted_text.endswith('\n'):
formatted_text = formatted_text[:-1]
return formatted_text
def assert_format(
source: str,
expected: str,
mode: black.Mode = DEFAULT_MODE,
*,
fast: bool = False,
minimum_version: Optional[Tuple[int, int]] = None,
) -> None:
"""Convenience function to check that Black formats as expected.
You can pass @minimum_version if you're passing code with newer syntax to guard
safety guards so they don't just crash with a SyntaxError. Please note this is
separate from TargetVerson Mode configuration.
"""
actual = black.format_str(source, mode=mode)
_assert_format_equal(expected, actual)
# It's not useful to run safety checks if we're expecting no changes anyway. The
# assertion right above will raise if reality does actually make changes. This just
# avoids wasted CPU cycles.
if not fast and source != expected:
# Unfortunately the AST equivalence check relies on the built-in ast module
# being able to parse the code being formatted. This doesn't always work out
# when checking modern code on older versions.
if minimum_version is None or sys.version_info >= minimum_version:
black.assert_equivalent(source, actual)
black.assert_stable(source, actual, mode=mode)
def _make_test(imports: Set[Union[str, Tuple[str, str]]], body: str) -> str:
# Discarding "builtins." and "__main__" probably isn't particularly useful
# for user code, but important for making a good impression in demos.
body = body.replace("builtins.", "").replace("__main__.", "")
if "st.from_type(typing." in body:
imports.add("typing")
imports |= {("hypothesis", "given"), ("hypothesis", "strategies as st")}
if " reject()\n" in body:
imports.add(("hypothesis", "reject"))
do_not_import = {"builtins", "__main__"}
direct = {
f"import {i}"
for i in imports - do_not_import if isinstance(i, str)
}
from_imports = defaultdict(set)
for module, name in {i for i in imports if isinstance(i, tuple)}:
from_imports[module].add(name)
from_ = {
"from {} import {}".format(module, ", ".join(sorted(names)))
for module, names in from_imports.items()
if module not in do_not_import
}
header = IMPORT_SECTION.format(
imports="\n".join(sorted(direct) + sorted(from_)))
nothings = body.count("st.nothing()")
if nothings == 1:
header += "# TODO: replace st.nothing() with an appropriate strategy\n\n"
elif nothings >= 1:
header += "# TODO: replace st.nothing() with appropriate strategies\n\n"
return black.format_str(header + body, mode=black.FileMode())
def convert_to_nodes(ref: metadsl.ExpressionReference) -> Nodes:
"""
Converts an expression into a node mapping.
"""
nodes: Nodes = []
for ref in ref.descendents:
node: typing.Union[CallNode, PrimitiveNode]
value = ref.expression
if isinstance(value, metadsl.Expression):
children = ref.children
func_str = function_or_type_repr(value.function)
node = CallNode(
id=str(ref.hash),
type_params=typevars_to_typeparams(
metadsl.typing_tools.get_fn_typevars(value.function))
or None,
function=black.format_str(
f"{func_str}\n{value._type_str}"
if SHOW_TYPES else func_str,
mode=black_file_mode,
),
args=[str(a) for a in children.args] or None,
kwargs={k: str(v)
for k, v in children.kwargs.items()} or None,
)
else:
node = PrimitiveNode(
id=str(ref.hash),
type=function_or_type_repr(type(value)),
repr=metadsl_str(value),
)
nodes.append(node)
return nodes
def run_black(src: Path, src_contents: str,
black_args: Dict[str, Union[bool, int]]) -> List[str]:
"""Run the black formatter for the Python source code given as a string
Return lines of the original file as well as the formatted content.
:param src: The originating file path for the source code
:param src_contents: The source code as a string
:param black_args: Command-line arguments to send to ``black.FileMode``
"""
config = black_args.pop("config", None)
defaults = read_black_config(src, config)
combined_args = {**defaults, **black_args}
effective_args = {}
if "line_length" in combined_args:
effective_args["line_length"] = combined_args["line_length"]
if "skip_string_normalization" in combined_args:
# The ``black`` command line argument is
# ``--skip-string-normalization``, but the parameter for
# ``black.FileMode`` needs to be the opposite boolean of
# ``skip-string-normalization``, hence the inverse boolean
effective_args["string_normalization"] = not combined_args[
"skip_string_normalization"]
# Override defaults and pyproject.toml settings if they've been specified
# from the command line arguments
mode = FileMode(**effective_args)
dst_contents = format_str(src_contents, mode=mode)
dst_lines: List[str] = dst_contents.splitlines()
return dst_lines
def generate_graph(n_clicks, n_submit, text):
if text is None:
return dash.no_update, dash.no_update
prompt = dedent(f"""
description: {desc}
code:
{code_exp}
description: {text}
code:
""").strip("\n")
response = openai.Completion.create(
engine="davinci",
prompt=prompt,
max_tokens=100,
stop=["description:", "code:"],
temperature=0,
)
output = response.choices[0].text.strip()
code = f"import plotly.express as px\nfig = {output}\nfig.show()"
formatted = black.format_str(code, mode=black.FileMode(line_length=50))
try:
fig = eval(output).update_layout(margin=dict(l=35, r=35, t=35, b=35))
except Exception as e:
fig = px.line(title=f"Exception: {e}. Please try again!")
return fig, f"```\n{formatted}\n```"
def wrap_python_code(code, max_length=100):
# this black currently does not split strings without spaces
while True:
new_code = code
for string_separator in '"', "'":
new_code = re.sub(
'(%s[0-9a-zA-Z\.\-\/\+]{%i,}?%s)' %
(string_separator, max_length + 1, string_separator),
lambda S: S.group(1)[:max_length] + string_separator + ' ' +
string_separator + S.group(1)[max_length:], new_code)
if new_code == code:
break
else:
code = new_code
logger.debug("\033[31mwrapped: %s\033[0m", code)
mode = black.Mode(
target_versions={black.TargetVersion.PY38},
line_length=max_length,
string_normalization=True,
experimental_string_processing=True,
)
# this will also ensure it's valid code
return black.format_str(code, mode=mode)
def generate_code(service_name: str, doc: bool = False) -> str:
model = load_service(service_name)
output = io.StringIO()
generate_service_types(output, model, doc=doc)
generate_service_api(output, model, doc=doc)
code = output.getvalue()
try:
import autoflake
import isort
from black import FileMode, format_str
# try to format with black
code = format_str(code, mode=FileMode(line_length=100))
# try to remove unused imports
code = autoflake.fix_code(code, remove_all_unused_imports=True)
# try to sort imports
code = isort.code(code,
config=isort.Config(profile="black",
line_length=100))
except Exception:
pass
return code
def write_metadata(metadata: Commands, version: str, tiny: bool = False) -> None:
path_in = Path(__file__).parent / ("serial.py.in.tiny" if tiny else "serial.py.in")
path_out = Path(__file__).parent / "serial.py"
tpl = ' {0} = "{0}"'
descriptions = pformat(
{**metadata["numeric"], **metadata["boolean"], **metadata["command"]}, width=160
)
numerics = "\n".join(tpl.format(key) for key in metadata["numeric"])
booleans = "\n".join(tpl.format(key) for key in metadata["boolean"])
commands = "\n".join(tpl.format(key) for key in metadata["command"])
all_commands = pformat(list(metadata["command"]))
kwargs = {
"version": version,
"descriptions": descriptions,
"numeric": numerics,
"boolean": booleans,
"command": commands,
"all_commands": all_commands,
}
with open(path_in.as_posix()) as fh:
output = fh.read().format(**kwargs)
output = black.format_str(output, 88)
with open(path_out.as_posix(), "w") as fh:
fh.write(output)
def blackformat(self):
"""Format code cells."""
import black
for cell in self.rawnb.cells:
fmt = cell["source"]
if cell["cell_type"] == "code":
try:
fmt = "\n".join(self.tag_magics(fmt))
has_semicolon = fmt.endswith(";")
fmt = black.format_str(
src_contents=fmt, mode=black.FileMode(line_length=79)
).rstrip()
if has_semicolon:
fmt += ";"
except Exception as ex:
log.info(ex)
fmt = fmt.replace(self.MAGIC_TAG, "")
cell["source"] = fmt
