def Load(cls, filename, inverted=False):
"""
Loads a dictionary from a file.
:param unicode filename:
Path and filename for source file.
:param bool inverted:
If True inverts the key/value order.
:returns dict:
Dictionary that was loaded
"""
from zerotk.easyfs import GetFileContents
contents = GetFileContents(filename)
contents_dict = dict()
import re
for line in contents.split('\n'):
search_result = re.search('(.*?)\s*=\s*(.*)', line)
if search_result is not None:
key, value = search_result.groups()
if inverted:
contents_dict[value] = key
else:
contents_dict[key] = value
return contents_dict
def testStringDictIOOrderAndSorted(embed_data):
filename = embed_data['dict.txt']
ordered_dict = OrderedDict()
ordered_dict['z'] = 'first'
ordered_dict['a'] = 'last'
# Test order
StringDictIO.Save(ordered_dict, filename)
assert GetFileContents(filename) == 'z = first\na = last\n'
# Test sorted
StringDictIO.Save(ordered_dict, filename, sort_items=True)
assert GetFileContents(filename) == 'a = last\nz = first\n'
def __init__(self, source=None, filename=None):
if source is None:
# Stores the original loaded sources into __source.
# This variable must be updated if we happen to save the file with
# any changes.
self.__original_source = GetFileContents(filename,
newline='',
encoding='UTF-8')
else:
self.__original_source = source
file_size = len(self.__original_source)
if file_size > self.MAX_FILE_SIZE:
# Some big files make the Parse algorithm get stuck.
raise FileTooBigError(filename, file_size)
self.filename = filename
self.symbols = set()
self.names = set()
self.import_blocks = []
self.code = self._Parse(self.__original_source)
from ._visitor import ASTVisitor
visitor = ASTVisitor()
visitor.Visit(self.code)
self.module, self.symbols, self.import_blocks = visitor._module, visitor.symbols, visitor.import_blocks
def testFixIsFrozen(embed_data):
"""
General test for tbe "tf is-frozen" command.
"""
filename = embed_data['testFixIsFrozen.py']
data_dir = embed_data.get_data_dir()
original = """
import coilib50
from coilib50.basic import property
if coilib50.IsFrozen():
print "Frozen"
property.Create('')
"""
assert CreateFile(filename, dedent(original), encoding='UTF-8')
app.TestScript(
dedent("""
>terraformer fix-is-frozen %(filename)s
- %(filename)s
""" % locals()))
assert GetFileContents(filename, encoding='UTF-8') == dedent("""
from ben10 import property_
from ben10.foundation.is_frozen import IsFrozen
import coilib50
from coilib50.basic import property
if IsFrozen():
print "Frozen"
property_.Create('')
""")
def handle_code(self, token, after, context):
if after:
return []
code = self.__parser.parse_code(token.line)
code_class = code.__class__.__name__
if code_class == 'Assignment':
self.variables[six.text_type(code.left)] = literal_eval(code.right)
elif code_class == 'Def':
self.functions[six.text_type(code.name)] = \
Function(code.name, code.parameters, token.children)
token.children = []
return []
elif code_class == 'Include':
filename = self._eval(code.filename, context)
parser = PugParser()
input_contents = GetFileContents(self._find_file(filename))
token_tree = parser.tokenize(input_contents)
return self._handle_line_token(token_tree, self.variables)
elif code_class == 'ForLoop':
return [token.indentation + repr(code)]
return []
def test_rename(embed_data, line_tester):
from zerotk.easyfs import GetFileContents
def Doit(lines):
source = ''.join([i + '\n' for i in lines])
terra = TerraFormer(source=source)
changed = terra.ReorganizeImports(
refactor={
'StringIO.StringIO': 'io.StringIO',
'cStringIO.StringIO': 'io.StringIO',
'cStringIO': 'from io.StringIO',
'StringIO': 'from io.StringIO',
}
)
if changed:
for i_symbol in terra.module.Walk():
if i_symbol.PREFIX == 'USE' and i_symbol.name in ('cStringIO.StringIO', 'StringIO.StringIO'):
i_symbol.Rename('StringIO')
return terra.GenerateSource().splitlines()
line_tester.TestLines(
GetFileContents(embed_data['rename.txt'], encoding='UTF-8'),
Doit,
)
def test_fix_format_error(embed_data):
"""
Check _FixFormat error output (detailed traceback).
"""
filename = embed_data['test_fix_format_error.py']
data_dir = embed_data.get_data_dir()
original = """
import zulu
from bravo import Bravo
import alpha
class Alpha(object):
def Method(self)
pass
def Function():
pass
"""
assert CreateFile(filename, dedent(original), encoding='UTF-8')
app.TestScript(dedent("""
>>>terraformer fix-format --single-job --traceback-limit=0 %(data_dir)s
- %(filename)s: ERROR:
\b\b
On TerraForming.ReorganizeImports with filename: %(filename)s
Had problems parsing:
> import zulu
> from bravo import Bravo
> import alpha
>\b
> class Alpha(object):
>\b
> def Method(self)
> pass
>\b
> def Function():
> pass
bad input: type=4, value=u'\\r\\n', context=('', (7, 20))
--- * ---
""" % locals()).replace('\b', ' '),
input_prefix='>>>')
assert GetFileContents(filename, encoding='UTF-8') == dedent("""
import zulu
from bravo import Bravo
import alpha
class Alpha(object):
def Method(self)
pass
def Function():
pass
""")
def testStringDictIODictWithSpaces(embed_data):
filename = embed_data['dict_with_spaces.txt']
dict_with_spaces = {'key with spaces': 'value with spaces'}
StringDictIO.Save(dict_with_spaces, filename)
assert GetFileContents(filename) == 'key with spaces = value with spaces\n'
assert StringDictIO.Load(filename) == dict_with_spaces
def TestFixEncoding(original, expected, encoding, lineno=0):
filename = embed_data['testFixEncoding.py_']
assert CreateFile(filename, dedent(original), encoding=encoding)
app.TestScript(
dedent("""
>terraformer fix-encoding %(filename)s
- %(filename)s: %(encoding)s (line:%(lineno)s)
""" % locals()))
obtained = GetFileContents(filename, encoding='UTF-8')
assert obtained == dedent(expected)
def FixEncoding(console_, *sources):
"""
Fix python module files encoding, converting all non-ascii encoded files to UTF-8.
:param sources: List of directories or files to process.
"""
from zerotk.easyfs import CreateFile, GetFileContents
import io
def GetPythonEncoding(filename):
import re
# Read the file contents in a encoding agnostic way, after all we're
# trying to find out the file encoding.
with open(filename, mode='rb') as iss:
for i, i_line in enumerate(iss.readlines()):
if i > 10:
# Only searches the first lines for encoding information.
break
r = re.match(b'#.*coding[:=] *([\w\-\d]*)', i_line)
if r is not None:
return i, r.group(1)
return 0, None
for i_filename in _GetFilenames(sources, [PYTHON_EXT]):
try:
# Try to open using ASCII. If it fails means that we have a
# non-ascii file.
with io.open(i_filename, encoding='ascii') as iss:
iss.read()
except UnicodeDecodeError:
try:
line_no, encoding = GetPythonEncoding(i_filename)
if encoding is None:
console_.Print(
'%s: <red>UKNOWN</r> Please configure the file coding.' % i_filename)
continue
console_.Item('%s: %s (line:%s)' %
(i_filename, encoding, line_no))
lines = GetFileContents(
i_filename, encoding=encoding).split('\n')
del lines[line_no]
lines = ['# coding: UTF-8'] + lines
contents = '\n'.join(lines)
CreateFile(i_filename, contents, encoding='UTF-8',
eol_style=EOL_STYLE_UNIX)
except:
console_.Print('<red>%s: ERROR</>' % i_filename)
raise
def generate(filename, include_paths=()):
"""
Creates and HTML from the given PyPUGly filename.
:param str filename:
:param list(str) include_paths:
:return str:
"""
parser = PugParser()
input_contents = GetFileContents(filename)
token_tree = parser.tokenize(input_contents)
generator = HtmlGenerator(
[os.path.dirname(filename)] + list(include_paths)
)
return generator.generate(token_tree)
def testFixFormat(embed_data):
"""
General test for tbe "tf fix-format" command.
This is a smoke test for the command interaction since most of the real
testing is done on pytest_terra_former.py
"""
filename = embed_data['testFixFormat.py']
data_dir = embed_data.get_data_dir()
original = """
import zulu
from bravo import Bravo
import alpha
class Alpha(object):
def Method(self):
pass
def Function():
pass
"""
assert CreateFile(filename, dedent(original), encoding='UTF-8')
app.TestScript(
dedent("""
>terraformer fix-format --single-job %(data_dir)s
- %(filename)s: FIXED
""" % locals()))
assert GetFileContents(filename, encoding='UTF-8') == dedent("""
from bravo import Bravo
import alpha
import zulu
class Alpha(object):
def Method(self):
pass
def Function():
pass
""")
def testAddImportSymbol(embed_data):
"""
General test for the "tf add-import-symbol" command.
"""
filename = embed_data['testFixFormat.py']
data_dir = embed_data.get_data_dir()
original = """
import zulu
from bravo import Bravo
import alpha
class Alpha(object):
def Method(self):
pass
def Function():
pass
"""
assert CreateFile(filename, dedent(original), encoding='UTF-8')
app.TestScript(
dedent("""
>terraformer add-import-symbol --single-job "__future__.unicode_literals" %(filename)s
- %(filename)s: FIXED
""" % locals()))
assert GetFileContents(filename, encoding='UTF-8') == dedent("""
from __future__ import unicode_literals
from bravo import Bravo
import alpha
import zulu
class Alpha(object):
def Method(self):
pass
def Function():
pass
""")
def testReorganizeImports(embed_data, line_tester):
from zerotk.easyfs import GetFileContents
def Doit(lines):
source = ''.join([i + '\n' for i in lines])
terra = TerraFormer(source=source)
changed_ = terra.ReorganizeImports(
refactor={
'coilib50.basic.implements': 'etk11.foundation.interface',
'coilib50.basic.inter': 'etk11.foundation.interface',
'coilib50.multithreading': 'multithreading',
'before_refactor_alpha.Alpha': 'after_refactor.Alpha',
'before_refactor_bravo.Bravo': 'after_refactor.Bravo',
}
)
return terra.GenerateSource().splitlines()
line_tester.TestLines(
GetFileContents(embed_data['reorganize_imports.txt'], encoding='UTF-8'),
Doit,
)
def create_project_file(target_filename, template_filename, config):
from jinja2 import Template
template = Template(GetFileContents(template_filename))
CreateFile(target_filename,
template.render(**config) + '\n',
eol_style=EOL_STYLE_UNIX)
def FixIsFrozen(console_, *sources):
"""
Fix some pre-determinated set of symbols usage with the format:
<module>.<symbol>
Eg.:
from coilib50.basic import property
property.Create
---
from ben10 import property_
property_.Create
This was necessary for BEN-30. Today TerraFormer only acts on import-statements. We have plans
to also act on imported symbols usage.
:param sources: List of directories or files to process.
"""
from zerotk.easyfs import CreateFile, EOL_STYLE_UNIX, GetFileContents
FIND_REPLACE = [
('StringIO', 'StringIO', 'from io import StringIO'),
('cStringIO', 'StringIO', 'from io import StringIO'),
('coilib50.IsFrozen', 'IsFrozen',
'from ben10.foundation.is_frozen import IsFrozen'),
('coilib50.IsDevelopment', 'IsDevelopment',
'from ben10.foundation.is_frozen import IsDevelopment'),
('coilib50.SetIsFrozen', 'SetIsFrozen',
'from ben10.foundation.is_frozen import SetIsFrozen'),
('coilib50.SetIsDevelopment', 'SetIsDevelopment',
'from ben10.foundation.is_frozen import SetIsDevelopment'),
('coilib40.basic.IsInstance', 'IsInstance',
'from ben10.foundation.klass import IsInstance'),
]
PROPERTY_MODULE_SYMBOLS = [
'PropertiesDescriptor',
'Property',
'Create',
'CreateDeprecatedProperties',
'CreateForwardProperties',
'FromCamelCase',
'MakeGetName',
'MakeSetGetName',
'MakeSetName',
'ToCamelCase',
'Copy',
'DeepCopy',
'Eq',
'PropertiesStr',
]
for i_symbol in PROPERTY_MODULE_SYMBOLS:
FIND_REPLACE.append(
('property.%s' % i_symbol, 'property_.%s' %
i_symbol, 'from ben10 import property_'),
)
for i_filename in _GetFilenames(sources, [PYTHON_EXT]):
contents = GetFileContents(i_filename)
imports = set()
for i_find, i_replace, i_import in FIND_REPLACE:
if i_find in contents:
contents = contents.replace(i_find, i_replace)
imports.add(i_import)
if imports:
console_.Item(i_filename)
lines = contents.split('\n')
index = None
top_doc = False
for i, i_line in enumerate(lines):
if i == 0:
for i_top_doc in ("'''", '"""'):
if i_top_doc in i_line:
console_.Print('TOPDOC START: %d' % i, indent=1)
top_doc = i_top_doc
break
continue
elif top_doc:
if i_top_doc in i_line:
console_.Print('TOPDOC END: %d' % i, indent=1)
index = i + 1
break
continue
elif i_line.startswith('import ') or i_line.startswith('from '):
index = i - 1
break
elif i_line.strip() == '':
continue
console_.Print('ELSE: %d: %s' % (i, i_line))
index = i
break
assert index is not None
lines = lines[0:index] + list(imports) + lines[index:]
contents = '\n'.join(lines)
CreateFile(
i_filename,
contents,
eol_style=EOL_STYLE_UNIX,
encoding='UTF-8'
)