def format_param(
self,
parameter: Parameter,
target_indent: int,
inline_formatting: bool,
param_list: bool = True,
) -> str:
string_indent = TAB * target_indent
if inline_formatting:
target_indent = 0
val = str(parameter)
try:
# A snakemake parameter is syntactically like a function parameter
ast_parse(f"param({val})")
except SyntaxError:
raise InvalidParameterSyntax(f"{parameter.line_nb}{val}") from None
if inline_formatting or param_list:
val = " ".join(
val.rstrip().split("\n")
) # collapse strings on multiple lines
extra_spacing = 0
if param_list:
val = f"f({val})"
extra_spacing = 3
val = self.run_black_format_str(val, target_indent, extra_spacing)
if param_list:
match_equal = re.match(r"f\((.*)\)", val, re.DOTALL)
val = match_equal.group(1)
val = textwrap.dedent(val)
val = self.align_strings(val, target_indent)
result = ""
if not inline_formatting:
for comment in parameter.pre_comments:
result += f"{string_indent}{comment}\n"
result += val.strip("\n")
if param_list:
result += ","
post_comment_iter = iter(parameter.post_comments)
if parameter._has_inline_comment:
result += f"{COMMENT_SPACING}{next(post_comment_iter)}"
result += "\n"
for comment in post_comment_iter:
result += f"{string_indent}{comment}\n"
return result
def literal_eval(node_or_string):
if COMPATIBILITY_MODE:
_safe_names = {'None': None, 'True': True, 'False': False}
if is_string(node_or_string):
node_or_string = ast_parse(node_or_string, mode='eval')
if isinstance(node_or_string, ast.Expression):
node_or_string = node_or_string.node # pylint: disable=E1103
def _convert(node):
if isinstance(node, ast.Const) and \
(is_string(node.value) or
isinstance(node.value,
(int, float, long, complex))):
return node.value
elif isinstance(node, ast.Tuple):
return tuple([_convert(x) for x in node.nodes])
elif isinstance(node, ast.List):
return list([_convert(x) for x in node.nodes])
elif isinstance(node, ast.Dict):
return dict((_convert(k), _convert(v))
for k, v in node.items)
elif isinstance(node, ast.Name):
if node.name in _safe_names:
return _safe_names[node.name]
elif isinstance(node, ast.UnarySub):
return -_convert(node.expr)
raise ValueError('malformed string')
return _convert(node_or_string)
else:
return ast.literal_eval(node_or_string)
def _get_class_names(self, file):
try:
with open(file) as fd:
parsed_source = ast_parse(fd.read())
class_names = [n.name for n in ast_walk(parsed_source) if isinstance(n, ClassDef)]
except IOError, e:
raise ClassLoaderError('Error - Couldn\'t open : \'{:}\'. Reason : {:}'.format(file, e.strerror))
def parse_gpr(str_expr):
"""parse gpr into AST
Parameters
----------
str_expr : string
string with the gene reaction rule to parse
Returns
-------
tuple
elements ast_tree and gene_ids as a set
"""
str_expr = str_expr.strip()
if len(str_expr) == 0:
return None, set()
for char, escaped in replacements:
if char in str_expr:
str_expr = str_expr.replace(char, escaped)
escaped_str = keyword_re.sub("__cobra_escape__", str_expr)
escaped_str = number_start_re.sub("__cobra_escape__", escaped_str)
tree = ast_parse(escaped_str, "<string>", "eval")
cleaner = GPRCleaner()
cleaner.visit(tree)
eval_gpr(tree, set()) # ensure the rule can be evaluated
return tree, cleaner.gene_set
def parse_gpr(str_expr):
"""parse gpr into AST
Parameters
----------
str_expr : string
string with the gene reaction rule to parse
Returns
-------
tuple
elements ast_tree and gene_ids as a set
"""
str_expr = str_expr.strip()
if len(str_expr) == 0:
return None, set()
for char, escaped in replacements:
if char in str_expr:
str_expr = str_expr.replace(char, escaped)
escaped_str = keyword_re.sub("__cobra_escape__", str_expr)
escaped_str = number_start_re.sub("__cobra_escape__", escaped_str)
tree = ast_parse(escaped_str, "<string>", "eval")
cleaner = GPRCleaner()
cleaner.visit(tree)
eval_gpr(tree, set()) # ensure the rule can be evaluated
return tree, cleaner.gene_set
def parse_expr(src):
"""
Parse a string that represents a Python expression and return the
corresponding Expr instance.
"""
e, = ast_parse(src).body
assert isinstance(e, ast.Expr)
return to_expr(e.value)
def parse_stmt(src):
"""
Parse a string that represents a Python statement and return the
corresponding Expr instance.
"""
body = ast_parse(src).body
if len(body) == 1:
return to_stmt(body[0])
else:
return stmt.Block([to_stmt(x) for x in body])
def _parse_python(self, sys_tokens):
_replace_unk = self.pov_replace_unk
_unk_replace_s = self._UNK_REPLACE_STR
_fix_python_orig_bug = self._fix_python_orig_bug
normalized_tokens = []
for token in sys_tokens:
token = unescape(token)
if token == 'DCNL':
new_token = '\n'
space_after = False
elif token == 'DCSP':
new_token = '\t'
space_after = False
elif _replace_unk and token == '<unk>':
new_token = _unk_replace_s
space_after = True
elif _fix_python_orig_bug and token == '_':
# If token is '_', concat two tokens (remove spaces).
new_token = '_'
if normalized_tokens and normalized_tokens[-1] == ' ':
# Do not concat 'def _', because 'def _private_name' is a common pattern.
# Do concat 'class _', because 'class _private_name' is uncommon, and '__class__' is common.
if len(normalized_tokens
) <= 1 or normalized_tokens[-2] != 'def':
normalized_tokens.pop()
space_after = False
else:
new_token = token
if _fix_python_orig_bug:
# Concat '= =' to '=='.
space_after = new_token.isidentifier()
else:
space_after = True
normalized_tokens.append(new_token)
if space_after:
normalized_tokens.append(' ')
try:
ast_parse(''.join(normalized_tokens))
except SyntaxError:
return 0
else:
return 1
def _load_from_string(self, code):
tree = ast_parse(code)
if COMPATIBILITY_MODE:
walk(tree, self)
else:
self.visit(tree)
required = list(self.requires_list)
self.requires_list = []
for r in required:
self.requires_list.append(requirement_versioning(r))
def __init__(self, name, ignore, code, mean, std, from_child=False):
"""
Инициализация вычислимого поля.
Аргументы:
name --- имя поля, str
ignore --- игнорирование поля как входа в нейронную сеть
code --- исполняемый код для расчёта значения поля
mean --- среднее значение для Z-scoring
std --- стандартное квадратичное отклонение для Z-scoring, неотрицательное
Пример значения code:
df['name'] = df['name1'] + df['name2']
"""
super().__init__(name, FieldType.calc, ignore, True)
if not isinstance(code, str):
raise ValueError('code in calc field is expected to be str')
if not isinstance(mean, float):
raise ValueError('numeric field mean must be float')
if not isinstance(std, float):
raise ValueError('numeric field std must be float')
if std < 0:
raise ValueError('deviation cannot be negative')
try:
ast_parse(code)
except SyntaxError as e:
raise CalcCodeParseError(e.msg)
if not "df[\'" + name + "\']" in code and not 'df[\"' + name + '\"]' in code:
raise CalcCodeParseError('В коде отсутствует ссылка на df[\"' +
name + '\"]')
regex = r"(?<=df\[)[ ]*('|\")[a-zA-Z0-9_ ]+('|\")[ ]*(?=])"
matches = re.finditer(regex, code, re.MULTILINE)
for match in matches:
fragment = code[match.start():match.end()]
tname = fragment.strip()[1:-1]
if tname != name and tname not in Field.df:
raise CalcCodeParseError(
'В коде используется ссылка на неизвестное поле с именем '
+ tname)
self.mean = mean
self.std = std
self.code = code
Field.register(name, from_child)
def _get_class_names(self, file):
try:
with open(file) as fd:
parsed_source = ast_parse(fd.read())
class_names = [
node.name for node in ast_walk(parsed_source)
if isinstance(node, ClassDef)
]
except IOError, e:
raise ClassLoaderError(
'Error - Couldn\'t open : \'{0}\'. Reason : {1}'.format(
file, e.strerror))
def _get_class_names(self, filename):
class_names = []
try:
with open(filename) as fd:
parsed_source = ast_parse(fd.read().strip(self.ENCODING_DECLARATION))
class_names = [n.name for n in ast_walk(parsed_source) if isinstance(n, ClassDef)]
except IOError as e:
raise ClassLoaderError('Error - Couldn\'t open : \'{:}\'. Reason : {:}.'.format(filename, e.strerror))
except SyntaxError as e:
raise ClassLoaderError('Error - Couldn\'t parse \'{:}\'. Reason: {:}.'.format(filename, e))
return class_names
def code_blocks(code_str: str, include_line_numbers=True):
"""Extract code blocks from code_str
:param code_str: String containing valid python code
:return: A generator of valid python code blocks, extracted sequentially from code_str
>>> code_str = '''
... def foo(a, b):
... return a + b
...
... foo(
... a=1,
... b=2
... )
... '''
>>> blocks = list(code_blocks(code_str))
>>> len(blocks)
2
>>> lineno, block = blocks[0]
>>> lineno # note that line numbers start at 1 (not like lists that start at 0)
2
>>> print(block)
def foo(a, b):
return (a + b)
>>> lineno, block = blocks[1]
>>> lineno
5
>>> print(block)
foo(a=1, b=2)
"""
if include_line_numbers:
for ast_obj in ast_parse(code_str).body:
yield ast_obj.lineno, ast_unparse(ast_obj).strip()
else:
for ast_obj in ast_parse(code_str).body:
yield ast_unparse(ast_obj).strip()
def format_param(
self,
parameter: Parameter,
target_indent: int,
inline_formatting: bool,
single_param: bool = False,
) -> str:
if inline_formatting:
target_indent = 0
comments = f"\n{TAB * target_indent}".join(parameter.comments)
val = str(parameter)
try:
# A snakemake parameter is syntactically like a function parameter
ast_parse(f"param({val})")
except SyntaxError:
raise InvalidParameterSyntax(f"{parameter.line_nb}{val}") from None
if inline_formatting:
val = val.replace("\n", "") # collapse strings on multiple lines
try:
val = self.run_black_format_str(val, target_indent)
except InvalidPython:
if "**" in val:
val = val.replace("** ", "**")
val = self.align_strings(val, target_indent)
if parameter.has_a_key(): # Remove space either side of '='
match_equal = re.match("(.*?) = (.*)", val, re.DOTALL)
val = f"{match_equal.group(1)}={match_equal.group(2)}"
val = val.strip("\n")
if single_param:
result = f"{val}{comments}\n"
else:
result = f"{val},{comments}\n"
return result
def parse_gpr(str_expr):
"""parse gpr into AST
returns: (ast_tree, {gene_ids})"""
str_expr = str_expr.strip()
if len(str_expr) == 0:
return None, set()
for char, escaped in replacements:
if char in str_expr:
str_expr = str_expr.replace(char, escaped)
escaped_str = keyword_re.sub("__cobra_escape__", str_expr)
escaped_str = number_start_re.sub("__cobra_escape__", escaped_str)
tree = ast_parse(escaped_str, "<string>", "eval")
cleaner = GPRCleaner()
cleaner.visit(tree)
eval_gpr(tree, set()) # ensure the rule can be evaluated
return tree, cleaner.gene_set
def _main_ExecCell(self, exec_msg):
"""
Executes a multi-line block of code.
"""
sid = exec_msg.hdr.sid
send_q = self._send_q
self._mod_msg._send_q = send_q
self._mod_msg._exec_id = id
self._stdout = QueueFileOut(send_q, msg.Stdout, sid)
self._stderr = QueueFileOut(send_q, msg.Stderr, sid)
self._stdin_q = Queue()
self._stdin = QueueFileIn(self._stdin_q, send_q, sid, exec_msg["echo_stdin"])
self._mod_sys.stdout = self._stdout
self._mod_sys.stderr = self._stderr
self._mod_sys.stdin = self._stdin
# name = exec_msg["name", "__cell_%s__" % (id,))
# mod = self._loader.new_module(name)
# self._globals.update(mod.__dict__)
fname = "cell_%d.py" % (exec_msg["cid"],)
try:
# apply source and ast transformations
source = transform_source(exec_msg, self._globals)
# write to file so that introspection works
f = open(fname, "w")
f.write(source)
f.close()
# self._loader.set_source(name, source)
source_ast = ast_parse(source, filename=fname, mode="exec")
source_ast = transform_ast(exec_msg, source_ast, source, self._globals)
# compile and execute
code = compile(source_ast, fname, "exec")
# self._loader.set_code(name, code)
exec_msg["transformed_source"] = source
self._globals["__exec_msg__"] = exec_msg
exec code in self._globals
except:
send_q.put(_get_except_msg(exec_msg))
finally:
send_q.put(msg.Done().as_reply_to(exec_msg))
def parse_gpr(str_expr):
"""parse gpr into AST
returns: (ast_tree, {gene_ids})"""
str_expr = str_expr.strip()
if len(str_expr) == 0:
return None, set()
for char, escaped in replacements:
if char in str_expr:
str_expr = str_expr.replace(char, escaped)
escaped_str = keyword_re.sub("__cobra_escape__", str_expr)
escaped_str = number_start_re.sub("__cobra_escape__", escaped_str)
tree = ast_parse(escaped_str, "<string>", "eval")
cleaner = GPRCleaner()
cleaner.visit(tree)
eval_gpr(tree, set()) # ensure the rule can be evaluated
return tree, cleaner.gene_set
def __init__(self, file):
self.file = file
with open(file, mode='r', encoding='utf-8-sig') as f:
text = f.read()
self.root = ast_parse(text)
