def format_blueprint_command(bp_file):
path = pathlib.Path(bp_file)
LOG.debug("Formatting blueprint {} using black".format(path))
if format_file_in_place(
path, fast=False, mode=FileMode(), write_back=WriteBack.DIFF
):
LOG.info("Patching above diff to blueprint - {}".format(path))
format_file_in_place(
path, fast=False, mode=FileMode(), write_back=WriteBack.YES
)
LOG.info("All done!")
else:
LOG.info("Blueprint {} left unchanged.".format(path))
def __init__(self):
self._mode = FileMode.from_configuration(
py36=False,
pyi=False,
skip_string_normalization=False,
skip_numeric_underscore_normalization=False,
)
def dag_to_airflow(self, output_dir, dag):
"""Generate the Airflow DAG representation for the provided DAG."""
output_file = Path(output_dir) / (dag.name + ".py")
formatted_dag = format_file_contents(
dag.to_airflow_dag(self), fast=False, mode=FileMode()
)
output_file.write_text(formatted_dag)
def main(args):
"""
This matches the default behaviour of the black formatter.
This script will raise an exception if changes by black are required.
"""
sources = set()
root = Path(os.getcwd())
p = Path(".")
include_regex = re_compile_maybe_verbose(DEFAULT_INCLUDES)
exclude_regex = re_compile_maybe_verbose(DEFAULT_EXCLUDES)
report = Report()
sources.update(
gen_python_files_in_dir(p, root, include_regex, exclude_regex, report,
get_gitignore(root)))
# To conform to flake8 line length used for this project.
mode = FileMode(line_length=79)
write_back = WriteBack.from_configuration(check=False, diff=not args.force)
reformat_many(
sources=sources,
fast=False,
write_back=write_back,
mode=mode,
report=report,
)
if report.change_count != 0:
exception_msg = """
Black formatter suggests formatting changes required
Run with '-f' option to automatically format.
"""
raise Exception(exception_msg)
def run_black(src: Path, src_contents: str,
black_args: Dict[str, Union[bool, int]]) -> List[str]:
"""Run the black formatter for the Python source code given as a string
Return lines of the original file as well as the formatted content.
:param src: The originating file path for the source code
:param src_contents: The source code as a string
:param black_args: Command-line arguments to send to ``black.FileMode``
"""
config = black_args.pop("config", None)
defaults = read_black_config(src, config)
combined_args = {**defaults, **black_args}
effective_args = {}
if "line_length" in combined_args:
effective_args["line_length"] = combined_args["line_length"]
if "skip_string_normalization" in combined_args:
# The ``black`` command line argument is
# ``--skip-string-normalization``, but the parameter for
# ``black.FileMode`` needs to be the opposite boolean of
# ``skip-string-normalization``, hence the inverse boolean
effective_args["string_normalization"] = not combined_args[
"skip_string_normalization"]
# Override defaults and pyproject.toml settings if they've been specified
# from the command line arguments
mode = FileMode(**effective_args)
dst_contents = format_str(src_contents, mode=mode)
dst_lines: List[str] = dst_contents.splitlines()
return dst_lines
def generate_code(service_name: str, doc: bool = False) -> str:
model = load_service(service_name)
output = io.StringIO()
generate_service_types(output, model, doc=doc)
generate_service_api(output, model, doc=doc)
code = output.getvalue()
try:
import autoflake
import isort
from black import FileMode, format_str
# try to format with black
code = format_str(code, mode=FileMode(line_length=100))
# try to remove unused imports
code = autoflake.fix_code(code, remove_all_unused_imports=True)
# try to sort imports
code = isort.code(code,
config=isort.Config(profile="black",
line_length=100))
except Exception:
pass
return code
def format(root, fast):
"""Formats all python files in basho/ with black"""
click.echo("Black is run against basho/")
from black import format_file_contents, FileMode, NothingChanged
from os import walk, path
directory = path.join(root, "basho/")
changes = [0, 0]
for root, subdirs, files in walk(directory):
for filename in files:
file_path = path.join(root, filename)
if file_path[-3:] != ".py":
continue
with open(file_path, "r") as f:
try:
out = format_file_contents(f.read(),
fast=fast,
mode=FileMode())
changes[0] += 1
with open(file_path, "w") as f:
f.write(out)
except NothingChanged:
changes[1] += 1
click.echo(f"{changes[0]} changed/ {changes[1]} not changed")
def to_dash(self, string, blacken=True, **kwargs):
blocks = self.parse_blocks(string)
components = self.blocks_to_components(blocks)
layout = f",\n{self.indent}".join(c for c in components)
dash_app = f"""\
import dash_html_components as html
import dash_core_components as dcc
from dash import Dash
from balderdash import load_dash_app
{self.precode}
# will be run before all sub apps
APP_PRECODE = \"""{self.app_precode}\"""
app = Dash(__name__)
app.config.suppress_callback_exceptions = True
app.layout = html.Div(
[
{layout}
]
)
if __name__ == "__main__":
app.run_server(debug=True)
"""
return format_str(dash_app, mode=FileMode())
def code(objeto):
"""
Genera un _code fencing_ en Markdown y ordena su visualización
Se espera que el *objeto* tenga asociado un archivo con su
código fuente. El resultado puede diferir del código literal ya
que se analizará el árbol sintáctico abstracto, se eliminarán
los _docstring_ y se reconstruirá el código. Se creará un salida
del tipo :class:`IPython.core.display.Markdown`
:param objeto: entidad de la que se desea el código fuente
:type objeto: object
"""
tree = parse(getsource(objeto))
for node in walk(tree):
try:
node.body = list(
filter(
lambda x: not isinstance(x, Expr) or not isinstance(
x.value, Str),
node.body,
))
except BaseException:
pass
display(
Markdown("```python\n" +
format_str(unparse(tree).strip(), mode=FileMode()) + "\n```"))
def write_python_file(self, file_path: str, source: str) -> None:
"""
Helper to write a Python source file.
:param file_path: Path of the file to write.
:param source: Content of the file to write.
"""
# If pretty-printing failed (the generated code has invalid Python
# syntax), write the original code anyway, and re-raise the
# SyntaxError in order to ease debugging.
exc: Optional[Exception] = None
if self.pretty_print:
try:
from black import FileMode, format_file_contents
source = format_file_contents(source,
fast=True,
mode=FileMode())
except ImportError:
check_source_language(
False,
"Black not available, not pretty-printing Python code",
severity=Severity.warning,
ok_for_codegen=True,
)
except SyntaxError as _exc:
exc = _exc
self.write_source_file(file_path, source, self.post_process_python)
if exc:
raise exc
def _format_module_str(text: str, is_pyi=False) -> str:
"""Apply black and isort formatting to text."""
from black import FileMode, format_str
from isort.api import sort_code_string
text = sort_code_string(text, profile="black", float_to_top=True)
text = format_str(text, mode=FileMode(line_length=79, is_pyi=is_pyi))
return text.replace("NoneType", "None")
def render_object(definition: dict,
template_file="shared/python/objects.j2") -> str:
env = build_jinja_environment()
template = env.get_template(template_file)
objects = objects_py(definition["yaml"])
rendered = template.render(objects=objects,
comments=definition["comments"])
return format_str(rendered, mode=FileMode())
def run_black(src: Path) -> Tuple[List[str], List[str]]:
"""Run the black formatter for the contents of the given Python file
Return lines of the original file as well as the formatted content.
"""
src_contents = src.read_text()
dst_contents = format_str(src_contents, mode=FileMode())
return src_contents.splitlines(), dst_contents.splitlines()
def format_with_black_to_clipboard(clipboard_content):
clipboard_content = clipboard_content.lstrip("%pyspark-format")
try:
res = format_str(clipboard_content, mode=FileMode(line_length=80))
res = "%pyspark\n" + res
notify("done!")
return res
except Exception as e:
notify("oops\n" + str(e))
return None
def write_model(self):
app = apps.get_app_config(self.app_name)
file_path = Path(app.models_module.__file__)
with file_path.open("a") as model_file:
model_file.write(f"\n\n{self.render()}")
# Format file with black to make stuff nicer (a.k.a. I'm lazy)
format_file_in_place(file_path,
fast=False,
mode=FileMode(),
write_back=WriteBack.YES)
def decode_code(code):
try:
code = b64decode(code).decode("utf-8")
tree = ast.parse(code)
parsed = astunparse.unparse(tree).strip()
cleaned = re.sub(r"^\s*\n", "", parsed, flags=re.MULTILINE)
blacked = format_str(cleaned, mode=FileMode())
return blacked
except Exception as e:
exc.append(type(e))
return ""
def gen(code: str, globals: Dict = None, locals: Dict = None) -> str:
"""
A wrapper of builtin `exec` function.
"""
try:
from black import FileMode, format_str
code = format_str(code, mode=FileMode(line_length=100))
except Exception:
pass
exec(code, globals, locals)
return code
def format_code_cells(notebobk):
file_mode = FileMode()
for cell in notebobk.cells:
if cell["cell_type"] == "code":
tags = cell.get("metadata", dict()).get("tags", [])
if "not-formatted" not in tags:
try:
result = format_str(cell["source"], mode=file_mode)
if result[-1] == "\n" and cell["source"] != "\n":
result = result[:-1]
cell["source"] = result
except InvalidInput:
pass
def create_bp_dir(bp_cls=None, bp_dir_name=None, with_secrets=False):
bp_dir_name = bp_dir_name or bp_cls.__name__
dir_name = os.getcwd()
LOG.info("Creating blueprint directory")
bp_dir, _, _, _ = init_bp_dir(dir_name, bp_dir_name)
LOG.info("Rendering blueprint file template")
bp_data = render_bp_file_template(bp_cls, with_secrets)
LOG.info("Formatting blueprint file using black")
bp_data = format_str(bp_data, mode=FileMode())
LOG.info("Creating blueprint file")
create_bp_file(bp_dir, bp_data)
def handler(event, context):
if "body" in event and event["body"]:
params = json.loads(event["body"])
else:
params = event
try:
git_repo = params["git_repo"]
git_branch = params[
"git_branch"] if "git_branch" in params else "master"
except KeyError:
return {
"body": json.dumps({"message": "git_repo is not provided"}),
"headers": {
"Content-Type": "application/json"
},
"statusCode": 401,
}
try:
r = requests.get(f"{git_repo}/archive/{git_branch}.zip")
open(f"{LOCAL_PATH}/{git_branch}.zip", 'wb').write(r.content)
with ZipFile(f"{LOCAL_PATH}/{git_branch}.zip", "r") as zipObj:
zipObj.extractall(f"{LOCAL_PATH}")
for root, dirs, files in os.walk(LOCAL_PATH):
for file in files:
if file.endswith(".py"):
format_file_in_place(Path(f"{root}/{file}"), False,
FileMode(), WriteBack.DIFF)
return {
"body": {
"message": "result in logs"
},
"headers": {
"Content-Type": "application/json"
},
"statusCode": 200,
}
except Exception as e:
message = e.message if hasattr(e, 'message') else e
return {
"body": {
"message": message
},
"headers": {
"Content-Type": "application/json"
},
"statusCode": 200,
}
def pretty_print(code):
if not self.pretty_print:
return code
try:
from black import FileMode, format_file_contents
return format_file_contents(code, fast=True, mode=FileMode())
except ImportError:
check_source_language(
False,
'Black not available, not pretty-printing Python code',
severity=Severity.warning,
ok_for_codegen=True)
return code
def generate_models(input_sql: str, generator: str, output_file: str) -> None:
file_content: str = get_file_content(filename=input_sql)
raw_sql_commands: List[str] = parse_commands(content=file_content)
context: Context = create_context(raw_sql_commands)
remmaper, output_template = GENERATOR_CONFIGS[generator]
remmaper.remap(context)
models_text = generate(output_template, context)
formatted_models_text = format_str(models_text, mode=FileMode())
with open(output_file, "w") as output:
output.write(formatted_models_text)
def black(self, line, cell):
"""Magic command to format the IPython cell."""
args = magic_arguments.parse_argstring(self.black, line)
line_length = args.line_length
if cell:
try:
from black import FileMode
mode = FileMode(line_length=line_length)
formatted = format_str(src_contents=cell, mode=mode)
except TypeError:
formatted = format_str(src_contents=cell,
line_length=line_length)
if formatted and formatted[-1] == "\n":
formatted = formatted[:-1]
self.shell.set_next_input(formatted, replace=True)
def format_code(code: str, **kwargs):
mode = FileMode(
target_versions=kwargs.get("versions", set()),
line_length=kwargs.get("line_length", DEFAULT_LINE_LENGTH),
is_pyi=kwargs.get("pyi", False),
string_normalization=kwargs.get("skip_string_normalization", True),
)
try:
formatted = format_str(code, mode=mode)
except NothingChanged as err:
LOGGER.debug(repr(err))
return ""
else:
LOGGER.debug("formatted: %s\n", formatted)
return formatted
def black(self, line, cell):
args = magic_arguments.parse_argstring(self.black, line)
# More info about the `--target-version` option: https://github.com/psf/black/issues/751
mode = FileMode(
line_length=args.line_length,
string_normalization=not args.string_normalization,
)
formatted_cell = format_str(cell, mode=mode)
formatted_cell = (formatted_cell[:-1]
if not args.final_newline and formatted_cell
and formatted_cell[-1] == "\n" else formatted_cell)
self.shell.set_next_input(formatted_cell, replace=True)
def create_bp_dir(bp_cls=None, bp_dir=None, with_secrets=False, metadata_obj=None):
if not bp_dir:
bp_dir = os.path.join(os.getcwd(), bp_cls.__name__)
LOG.info("Creating blueprint directory")
_, _, _, _ = init_bp_dir(bp_dir)
LOG.info("Rendering blueprint file template")
bp_data = render_bp_file_template(
cls=bp_cls, with_secrets=with_secrets, metadata_obj=metadata_obj
)
LOG.info("Formatting blueprint file using black")
bp_data = format_str(bp_data, mode=FileMode())
LOG.info("Creating blueprint file")
create_bp_file(bp_dir, bp_data)
def render_actions_as_python(
actions: List[dict],
template_file="shared/python/actions_referentiel.j2") -> str:
"""Render all actions into a single python file."""
env = build_jinja_environment()
def add_points(actions: List[dict]):
for action in actions:
action["points"] = action.get("points", None)
add_points(action["actions"])
add_points(actions)
template = env.get_template(template_file)
rendered = template.render(actions=actions)
return format_str(rendered, mode=FileMode())
def create_pipeline_file(self, pipeline, pipeline_export_format,
pipeline_export_path, pipeline_name):
self.log.info('Creating pipeline definition as a .' +
pipeline_export_format + ' file')
if pipeline_export_format == "json":
with open(pipeline_export_path, 'w', encoding='utf-8') as file:
json.dump(pipeline_export_path,
file,
ensure_ascii=False,
indent=4)
else:
# Load template from installed elyra package
loader = PackageLoader('elyra', 'templates/airflow')
template_env = Environment(loader=loader)
template_env.filters['regex_replace'] = lambda string: re.sub(
"[-!@#$%^&*(){};:,/<>?|`~=+ ]", "_", string) # nopep8 E731
template = template_env.get_template('airflow_template.jinja2')
notebook_ops = self._cc_pipeline(pipeline, pipeline_name)
runtime_configuration = self._get_metadata_configuration(
namespace=MetadataManager.NAMESPACE_RUNTIMES,
name=pipeline.runtime_config)
user_namespace = runtime_configuration.metadata.get(
'user_namespace') or 'default'
cos_secret = runtime_configuration.metadata.get('cos_secret')
description = f"Created with Elyra {__version__} pipeline editor using `{pipeline.source}`."
python_output = template.render(operations_list=notebook_ops,
pipeline_name=pipeline_name,
namespace=user_namespace,
cos_secret=cos_secret,
kube_config_path=None,
is_paused_upon_creation='False',
in_cluster='True',
pipeline_description=description)
# Write to python file and fix formatting
with open(pipeline_export_path, "w") as fh:
autopep_output = autopep8.fix_code(python_output)
output_to_file = format_str(autopep_output, mode=FileMode())
fh.write(output_to_file)
return pipeline_export_path
def gen(code: str,
globals: Dict = None,
locals: Dict = None,
cls: Type = None) -> str:
"""
Customized `exec` function.
"""
try:
from black import format_str, FileMode
code = format_str(code, mode=FileMode(line_length=100))
except Exception:
pass
for_class = 'for ' + cls.__name__ if cls else ''
logger.debug(f'Generating {for_class} ...\n{code}')
exec(code, globals, locals)
return code
def generate(service: str, doc: bool, save: bool):
"""
Generate types and API stubs for a given AWS service.
SERVICE is the service to generate the stubs for (e.g., sqs, or cloudformation)
"""
from click import ClickException
try:
model = load_service(service)
except UnknownServiceError:
raise ClickException("unknown service %s" % service)
output = io.StringIO()
generate_service_types(output, model, doc=doc)
generate_service_api(output, model, doc=doc)
code = output.getvalue()
try:
# try to format with black
from black import FileMode, format_str
code = format_str(code, mode=FileMode())
except Exception:
pass
if not save:
# either just print the code to stdout
click.echo(code)
return
# or find the file path and write the code to that location
here = os.path.dirname(__file__)
path = os.path.join(here, "api", service)
if not os.path.exists(path):
click.echo("creating directory %s" % path)
mkdir(path)
file = os.path.join(path, "__init__.py")
click.echo("writing to file %s" % file)
with open(file, "w") as fd:
fd.write(code)
click.echo("done!")
