def test_add_strings_compiles_fine(self):
migration = FileMigration('filename', 'doesnt-matter')
m1 = StringMigration('first', 'πρώτο')
m2 = StringMigration('\nsecond', '\nδεύτερο')
m3 = StringMigration('\nêà', '\nεα')
migration.add_string(m1)
migration.add_string(m2)
migration.add_string(m3)
assert migration.modified_strings == [m1, m2, m3]
assert migration.compile() == 'πρώτο\nδεύτερο\nεα'
def test_revert_strings(self):
migration = FileMigration('filename', 'doesnt-matter')
m1 = StringMigration('first', 'πρώτο')
m2 = StringMigration('second', 'δεύτερο')
m3 = StringMigration('third', 'τρίτο')
migration.add_string(m1)
migration.add_string(m2)
migration.add_string(m3)
migration.revert()
assert migration.compile() == 'firstsecondthird'
assert migration.modified_strings == []
def _parse_token(self, token, parser, original_string):
"""Parse the given token and create a migration object for a particular
substring of the whole template code.
:param django.template.base.Token token: holds information on a
specific line in the template, e.g. contains 'trans "A string"'
:param django.template.base.Parser parser: the parser object that holds
information on the whole template document
:param unicode original_string: the full string matched in the
template, e.g. '{% trans "A string" %}'
:return: a StringMigration object that contains information about
the Django and Transifex Native syntax of a specific set of strings
matched in the template
:rtype: StringMigration
"""
# Simple text found
if token.token_type == TOKEN_TEXT:
return self._parse_text(token, original_string)
# A comment tag was found, so we need to retrieve the comment text
# e.g. {# Translators: Do this and that #}
# If it's a generic (non-translator) comment, ignore
elif token.token_type == TOKEN_COMMENT:
comment = _retrieve_comment(token.contents)
if comment:
self._comment = comment
self._current_string_migration = StringMigration(
original_string, '')
return None, None
# A variable was found; copy as is
elif token.token_type == TOKEN_VAR:
# that's a special case we need to take care of:
# {{ _("Are you sure you want to remove the ($(collaborator_count)) selected collaborators?")|escapejs }} # noqa
if token.contents.startswith('_('):
token.token_type = TOKEN_BLOCK
clos_par_pos = 0
for i, j in enumerate(token.contents):
if j == ')':
clos_par_pos = i
token.contents = ('trans ' + token.contents[2:clos_par_pos] +
token.contents[clos_par_pos + 1:])
else:
return StringMigration(original_string, original_string), None
return self._parse_block(token, parser, original_string)
def _final_string_migration(
self,
original_string,
new_string,
confidence=Confidence.HIGH,
consumed_tokens=None,
):
"""Create a migration object, taking into account any open comments.
:param unicode original_string:
:param unicode new_string:
:param int confidence: the confidence level of the migration
:param List[Token] consumed_tokens: a list of extra tokens that were
consumed for this migration; e.g. in a {% blocktrans %} token
it will contain all text inside this up to {% endblocktrans %}
"""
consumed_tokens = consumed_tokens or []
if self._current_string_migration:
self._current_string_migration.update(original_string, new_string)
string_migration = self._current_string_migration
self._current_string_migration = None
return string_migration, consumed_tokens
return (StringMigration(original_string,
new_string,
confidence=confidence), consumed_tokens)
def test_update_prepend(self):
migration = StringMigration('first', 'FIRST', Confidence.HIGH)
migration.update('\nsecond', '\nSECOND', append=False)
migration.update('\nthird', '\nTHIRD', Confidence.LOW, append=False)
assert migration.confidence == Confidence.LOW
assert migration.modified is True
assert migration.original == '\nthird\nsecondfirst'
assert migration.new == '\nTHIRD\nSECONDFIRST'
def test_update_append(self):
migration = StringMigration('first', 'FIRST', Confidence.HIGH)
migration.update('\nsecond', '\nSECOND')
assert migration.confidence == Confidence.HIGH
migration.update('\nthird', '\nTHIRD', Confidence.LOW)
assert migration.confidence == Confidence.LOW
assert migration.modified is True
assert migration.original == 'first\nsecond\nthird'
assert migration.new == 'FIRST\nSECOND\nTHIRD'
def _parse_text(self, token, original_string):
"""Parse a text token and return a migration object.
Text tokens are those that are not inside special syntax.
For example in the following template:
'''
<a href="...">Link</a>
{% anytag %}Some text{% endanytag %}
'''
there are 2 text tokens:
- '<a href="...">Link</a>\n\n'
- 'Some text'
:param django.template.base.token: the token object
:param unicode original_string: the string found in the template
:return: a tuple containing a StringMigration instance, if applicable,
and a list of extra tokens that were consumed
:rtype: Tuple[StringMigration, List[Token]]
"""
# If the previous tag was an opening {% comment %} tag,
# and now we have the text inside, e.g.
# {% comment %}My comment{% endcomment %}
# we're here: ^--------^
# so now we need to retrieve the actual comment
if self._comment == COMMENT_FOUND:
self._comment = _retrieve_comment(token.contents)
# Make sure to record that the tag is removed from the migrated
# result
if self._current_string_migration:
self._current_string_migration.update(original_string, '')
return None, None
# In any other case, just copy the content as is
else:
# String migration was already open, make sure to record
# that the tag is removed from the migrated result
if self._current_string_migration:
self._current_string_migration.update(original_string,
original_string)
return None, None
# No open string migration, return a new one
else:
return StringMigration(original_string, original_string), None
def add_in_between(txt_range):
"""Add the simple text found between the last migrated node
and the current node.
Ensures that the text that should remain untransformed
is included in the final file.
"""
if txt_range[0] > last_migrated_char:
try:
original_str = src[last_migrated_char:txt_range[0]]
file_migration.add_string(
StringMigration(
original=original_str,
new=original_str,
confidence=Confidence.HIGH,
))
except Exception as e:
Color.echo(
'[error]Error while adding in-between content '
'in [file]{}[end]. last_migrated_char={}.'
'Error: {}'.format(filename, last_migrated_char,
e))
raise
def _parse_block(self, token, parser, original_string):
"""Parse any {% ... %} token and return a migration object.
:param django.template.base.Token token: the token object
:param django.template.base.parser: the parser object
:param unicode original_string: the string found in the template
"""
# Split the given token into its parts (includes the template tag
# name), e.g. "{% trans "This is the title" context "Some context" %}"
# returns: ['trans', '"This is the title"', 'context', '"Some
# context"']
bits = token.split_contents()
tag_name = bits[0]
# Right after {% load i18n %} append a {% load transifex %} tag
if (tag_name == templates.LOAD_TAG
and bits[1] == templates.DJANGO_i18n_TAG_NAME):
# Make sure there is not already a tag that loads "transifex"
# by checking all remaining nodes
for t in parser.tokens:
if (templates.LOAD_TAG in t.contents
and templates.TRANSIFEX_TAG_NAME in t.contents):
return (StringMigration(original_string,
original_string), None)
string_migration = StringMigration(
original_string,
'original\n{% load transifex %}'.replace(
'original', original_string),
)
return string_migration, None
# A {% with %} tag was found
elif tag_name == templates.WITH_TAG:
with_kwargs = token_kwargs(bits[1:], parser, support_legacy=True)
self._with_kwargs.append(
{k: v.var
for k, v in with_kwargs.items()})
# An {% endwith %} tag was found
elif tag_name == templates.ENDWITH_TAG:
self._with_kwargs.pop()
# A {% comment %} tag was found; If this is a translation comment,
# expect the actual comment text to follow shortly
elif tag_name == templates.COMMENT_TAG:
next_token = parser.tokens[0] if parser.tokens else None
if next_token.token_type == TOKEN_TEXT:
comment = _retrieve_comment(next_token.contents)
if comment:
self._comment = COMMENT_FOUND
# Create a string migration and start keeping track
# of all the strings that will be migrated
# within the following set of tokens that apply
self._current_string_migration = StringMigration(
original_string, '')
return None, None
# An {% endcomment %} tag was found
elif tag_name == templates.ENDCOMMENT_TAG:
# No need to do anything special, just make sure to record
# that the tag is removed from the migrated result
# If a translation comment wasn't open, ignore it
if self._comment is not None:
if self._current_string_migration:
self._current_string_migration.update(original_string, '')
return None, None
# A {% trans %} tag was found
elif tag_name in templates.TRANSLATE_TAGS:
return self._parse_trans(token, parser, original_string)
# A {% blocktrans %} tag was found
elif tag_name in templates.BLOCK_TRANSLATE_TAGS:
return self._parse_blocktrans(token, parser, original_string)
# This is the default case; any other block tag that wasn't
# explicitly covered above, which means that it doesn't need migration
return StringMigration(original_string, original_string), None
class DjangoTagMigrationBuilder(object):
"""Parses Django templates and creates file migrations for each template.
A migration is an object that describes the changes that need to be made
in order to change a template file that uses the Django i18n syntax
to a file that uses the Transifex Native syntax.
This class is stateful, but it can be reused. The reason it is created
this way is for optimization, i.e. for not having to create new instances
of the class for each migrated file.
"""
def __init__(self):
self._reset()
def _reset(self):
"""Reset some state parameters, so that a new migration can be created.
The reason this method exists is for optimization, i.e. not having
to create new instances of the class for each file.
"""
# Each entry is a dictionary with var_name/var_value items,
# each representing a {% with var_name=var_value %} tag.
# Since {% with %} tags can be nested, this list is LIFO.
self._with_kwargs = []
# The current (developer) comment that was parsed, if any
# Comments in Django templates preceed the actual string token,
# so when we find one we need to keep it aside in order to use it
# when we parse the actual translatable string
self._comment = None
self._current_string_migration = None
# Sometimes, like when a comment is available, we cannot parse
# a translatable string in one go, so we need to create
# a StringMigration object and amend it as soon as we have all its
# information.
self._current_string_migration = None
def build_migration(self, src, filename=None, charset='utf-8'):
"""Create a migration for a Django template file to Transifex Native syntax.
The returned object contains every change separately, so that
it can be reviewed string by string.
:param unicode src: the whole Django template
:param str filename: the filename of the original template
:param str charset: the character set to use
:return: a FileMigration instance
:rtype: FileMigration
"""
self._reset()
src = force_text(src, charset)
# Using the DebugLexer because we need the positional information
# of each token (start/end pos). It is slower than Lexer, but Lexer
# doesn't provide that information
tokens = DebugLexer(src).tokenize()
parser = Parser(tokens, libraries={}, builtins=[], origin=filename)
# Since no template libraries are loaded when this code is running,
# we need to override the find function in order to use the
# functionality of the Parser class. The overridden function returns
# the object as given.
# Without the override, a KeyError would be raised inside the parser.
parser.find_filter = find_filter_identity
# Create a migration object for this template; we'll add stuff to it
# as we go
migration = FileMigration(filename, src)
while parser.tokens:
token = parser.next_token()
start, end = token.position
# Parse the current token. This may or may not return a migration.
# Also it may return a list of tokens that were consumed,
# additionally to the current token. If this happens,
# `_parse_token()` will have made sure that `parser` has moved
# forward, consuming those tokens, so that they don't appear again
# in the while loop.
string_migration, extra_consumed_tokens = self._parse_token(
token, parser, original_string=src[start:end])
if not string_migration:
continue
# If additional tokens were consumed, we need to add
# them in the migration, so that the StringMigration object
# includes the information of what part of the original template
# was migrated to the new syntax, for this particular translatable
# string
if extra_consumed_tokens:
for extra_token in extra_consumed_tokens:
start, end = extra_token.position
string_migration.update(src[start:end], '')
migration.add_string(string_migration)
return migration
def _parse_token(self, token, parser, original_string):
"""Parse the given token and create a migration object for a particular
substring of the whole template code.
:param django.template.base.Token token: holds information on a
specific line in the template, e.g. contains 'trans "A string"'
:param django.template.base.Parser parser: the parser object that holds
information on the whole template document
:param unicode original_string: the full string matched in the
template, e.g. '{% trans "A string" %}'
:return: a StringMigration object that contains information about
the Django and Transifex Native syntax of a specific set of strings
matched in the template
:rtype: StringMigration
"""
# Simple text found
if token.token_type == TOKEN_TEXT:
return self._parse_text(token, original_string)
# A comment tag was found, so we need to retrieve the comment text
# e.g. {# Translators: Do this and that #}
# If it's a generic (non-translator) comment, ignore
elif token.token_type == TOKEN_COMMENT:
comment = _retrieve_comment(token.contents)
if comment:
self._comment = comment
self._current_string_migration = StringMigration(
original_string, '')
return None, None
# A variable was found; copy as is
elif token.token_type == TOKEN_VAR:
# that's a special case we need to take care of:
# {{ _("Are you sure you want to remove the ($(collaborator_count)) selected collaborators?")|escapejs }} # noqa
if token.contents.startswith('_('):
token.token_type = TOKEN_BLOCK
clos_par_pos = 0
for i, j in enumerate(token.contents):
if j == ')':
clos_par_pos = i
token.contents = ('trans ' + token.contents[2:clos_par_pos] +
token.contents[clos_par_pos + 1:])
else:
return StringMigration(original_string, original_string), None
return self._parse_block(token, parser, original_string)
def _parse_text(self, token, original_string):
"""Parse a text token and return a migration object.
Text tokens are those that are not inside special syntax.
For example in the following template:
'''
<a href="...">Link</a>
{% anytag %}Some text{% endanytag %}
'''
there are 2 text tokens:
- '<a href="...">Link</a>\n\n'
- 'Some text'
:param django.template.base.token: the token object
:param unicode original_string: the string found in the template
:return: a tuple containing a StringMigration instance, if applicable,
and a list of extra tokens that were consumed
:rtype: Tuple[StringMigration, List[Token]]
"""
# If the previous tag was an opening {% comment %} tag,
# and now we have the text inside, e.g.
# {% comment %}My comment{% endcomment %}
# we're here: ^--------^
# so now we need to retrieve the actual comment
if self._comment == COMMENT_FOUND:
self._comment = _retrieve_comment(token.contents)
# Make sure to record that the tag is removed from the migrated
# result
if self._current_string_migration:
self._current_string_migration.update(original_string, '')
return None, None
# In any other case, just copy the content as is
else:
# String migration was already open, make sure to record
# that the tag is removed from the migrated result
if self._current_string_migration:
self._current_string_migration.update(original_string,
original_string)
return None, None
# No open string migration, return a new one
else:
return StringMigration(original_string, original_string), None
def _parse_block(self, token, parser, original_string):
"""Parse any {% ... %} token and return a migration object.
:param django.template.base.Token token: the token object
:param django.template.base.parser: the parser object
:param unicode original_string: the string found in the template
"""
# Split the given token into its parts (includes the template tag
# name), e.g. "{% trans "This is the title" context "Some context" %}"
# returns: ['trans', '"This is the title"', 'context', '"Some
# context"']
bits = token.split_contents()
tag_name = bits[0]
# Right after {% load i18n %} append a {% load transifex %} tag
if (tag_name == templates.LOAD_TAG
and bits[1] == templates.DJANGO_i18n_TAG_NAME):
# Make sure there is not already a tag that loads "transifex"
# by checking all remaining nodes
for t in parser.tokens:
if (templates.LOAD_TAG in t.contents
and templates.TRANSIFEX_TAG_NAME in t.contents):
return (StringMigration(original_string,
original_string), None)
string_migration = StringMigration(
original_string,
'original\n{% load transifex %}'.replace(
'original', original_string),
)
return string_migration, None
# A {% with %} tag was found
elif tag_name == templates.WITH_TAG:
with_kwargs = token_kwargs(bits[1:], parser, support_legacy=True)
self._with_kwargs.append(
{k: v.var
for k, v in with_kwargs.items()})
# An {% endwith %} tag was found
elif tag_name == templates.ENDWITH_TAG:
self._with_kwargs.pop()
# A {% comment %} tag was found; If this is a translation comment,
# expect the actual comment text to follow shortly
elif tag_name == templates.COMMENT_TAG:
next_token = parser.tokens[0] if parser.tokens else None
if next_token.token_type == TOKEN_TEXT:
comment = _retrieve_comment(next_token.contents)
if comment:
self._comment = COMMENT_FOUND
# Create a string migration and start keeping track
# of all the strings that will be migrated
# within the following set of tokens that apply
self._current_string_migration = StringMigration(
original_string, '')
return None, None
# An {% endcomment %} tag was found
elif tag_name == templates.ENDCOMMENT_TAG:
# No need to do anything special, just make sure to record
# that the tag is removed from the migrated result
# If a translation comment wasn't open, ignore it
if self._comment is not None:
if self._current_string_migration:
self._current_string_migration.update(original_string, '')
return None, None
# A {% trans %} tag was found
elif tag_name in templates.TRANSLATE_TAGS:
return self._parse_trans(token, parser, original_string)
# A {% blocktrans %} tag was found
elif tag_name in templates.BLOCK_TRANSLATE_TAGS:
return self._parse_blocktrans(token, parser, original_string)
# This is the default case; any other block tag that wasn't
# explicitly covered above, which means that it doesn't need migration
return StringMigration(original_string, original_string), None
def _parse_trans(self, token, parser, original_string):
"""Parse a {% trans %} token and return a migration object.
:param django.template.base.Token token: the token object
:param django.template.base.parser: the parser object
:param unicode original_string: the string found in the template
"""
# Use Django's do_translate() method to parse the token
trans_node = do_translate(parser, token)
confidence = Confidence.LOW if trans_node.noop else Confidence.HIGH
message_context = trans_node.message_context
# Our SDK supports filter expressions
text = trans_node.filter_expression.token
# Source strings that contain XML symbols should use 'ut'. We determine
# whether the string contains XML symbols by testing if an escaping
# attempt changes it in any way.
# eg `{% trans "a b" %}` => `{% t "a b" %}`
# `{% trans "<xml>a</xml> b" %}` => `{% ut "<xml>a</xml> b" %}`
if isinstance(trans_node.filter_expression.var, string_types):
literal = trans_node.filter_expression.var
else:
literal = trans_node.filter_expression.var.literal
if (isinstance(literal, string_types)
and escape_html(literal) != literal):
tag_name = "ut"
else:
tag_name = "t"
params = {'_context': message_context, '_comment': self._comment}
# Reset the stored comment, so that it doesn't leak to the next token
self._comment = None
# Render the final output
t_tag = ['{%', tag_name, text, _render_params(params)]
if trans_node.asvar:
t_tag.extend(['as', trans_node.asvar])
t_tag.append('%}')
t_tag = ' '.join((thing.strip() for thing in t_tag if thing.strip()))
return self._final_string_migration(original_string,
t_tag,
confidence=confidence)
def _parse_blocktrans(self, token, parser, original_string):
"""Parse a {% blocktrans %} token and return a migration object.
:param django.template.base.Token token: the token object
:param django.template.base.parser: the parser object
:param unicode original_string: the string found in the template
"""
# Use Django's blocktranslate tag function to actually parse
# the whole tag, so that we easily get all information
# Internally, the do_block_translate() call will make the parser
# go forward, so the call to parser.next_token() will skip
# all tokens until {% endblocktrans %} (inclusive).
consumed_tokens = []
for t in parser.tokens: # these are just the remaining tokens
consumed_tokens.append(t)
if t.contents in templates.ENDBLOCK_TRANSLATE_TAGS:
break # we assume there will be a {% endblocktrans %} token
blocktrans_node = do_block_translate(parser, token)
message_context = blocktrans_node.message_context
singular_text = _render_var_tokens(blocktrans_node.singular)
plural_text = _render_var_tokens(blocktrans_node.plural)
# Start building the parameters supported by Transifex Native
params = {'_context': message_context, '_comment': self._comment}
# Plural support in Django works by using the "count" keyword
counter_var = blocktrans_node.countervar
if blocktrans_node.countervar:
params[counter_var] = blocktrans_node.counter.token
# Add any key/value pairs that hold placeholder/variable information
# e.g. {% blocktrans user.name as username %}
params.update({
key: value.token
for key, value in blocktrans_node.extra_context.items()
})
params = _render_params(params)
# Retrieve any variables inside text, e.g.
# "This is a {{ var }} and this is {{ another_var }}"
variables_in_text = (_get_variable_names(blocktrans_node.singular) +
_get_variable_names(blocktrans_node.plural))
# Reset the stored comment, so that it doesn't leak to the next token
self._comment = None
# Build the template of the tag for Transifex Native syntax
is_multiline = '\n' in singular_text or '\n' in plural_text
content = _make_plural(singular_text, plural_text, counter_var)
# Source strings that contain XML symbols should use 'ut'. We determine
# whether the string contains XML symbols by testing if an escaping
# attempt changes it in any way.
# eg `{% blocktrans %}a b{% endblocktrans %}` =>
# `{% t "a b" %}`
# eg `{% blocktrans %}<xml>a</xml> b{% endblocktrans %}` =>
# `{% ut "<xml>a</xml> b" %}`
if escape_html(content) != content:
tag_name = "ut"
else:
tag_name = "t"
has_apos, has_quot = "'" in content, '"' in content
use_block = is_multiline or (has_apos and has_quot)
if not use_block and has_quot:
surround_with = "'"
else:
surround_with = '"'
# Render the final output
t_tag = ['{% ', tag_name]
if not use_block:
t_tag.extend([' ', surround_with, content, surround_with])
if blocktrans_node.trimmed:
if use_block:
t_tag.append(' |trimmed')
else:
t_tag.append('|trimmed')
if params.strip():
t_tag.extend([' ', params])
if blocktrans_node.asvar:
t_tag.extend([' as ', blocktrans_node.asvar])
t_tag.append(' %}')
if use_block:
t_tag.extend([content, '{% end', tag_name, ' %}'])
t_tag = ''.join(t_tag)
# Determine the confidence of the migration
confidence = (Confidence.HIGH
if not variables_in_text else Confidence.LOW)
# Create the actual migration
return self._final_string_migration(original_string,
t_tag,
consumed_tokens=consumed_tokens,
confidence=confidence)
def _final_string_migration(
self,
original_string,
new_string,
confidence=Confidence.HIGH,
consumed_tokens=None,
):
"""Create a migration object, taking into account any open comments.
:param unicode original_string:
:param unicode new_string:
:param int confidence: the confidence level of the migration
:param List[Token] consumed_tokens: a list of extra tokens that were
consumed for this migration; e.g. in a {% blocktrans %} token
it will contain all text inside this up to {% endblocktrans %}
"""
consumed_tokens = consumed_tokens or []
if self._current_string_migration:
self._current_string_migration.update(original_string, new_string)
string_migration = self._current_string_migration
self._current_string_migration = None
return string_migration, consumed_tokens
return (StringMigration(original_string,
new_string,
confidence=confidence), consumed_tokens)
def _string(confidence=Confidence.HIGH):
"""Return a sample StringMigration object for testing."""
return StringMigration('original', 'new', confidence)
def transform(self, src, filename=None):
"""Parse the given Python file string and extract translatable content.
:param unicode src: a chunk of Python code
:param str filename: the filename of the code, i.e. the filename it
came from
:return: a list of SourceString objects
:rtype: list
"""
# Replace utf-8 magic comment, to avoid getting a
# "SyntaxError: encoding declaration in Unicode string"
src = ENCODING_PATTERN.sub('# ', src)
try:
tree = ast.parse(src)
visitor = CallDetectionVisitor([(x['modules'], x['function'])
for x in self._functions])
visitor.visit(tree)
except Exception as e:
Color.echo('[error]Error while parsing content '
'of [file]{}[end]: {}'.format(filename, e))
# Store an exception for this particular file
self.errors.append((filename, e))
return None
else:
attree = asttokens.ASTTokens(src, tree=tree)
file_migration = FileMigration(filename, src)
last_migrated_char = 0
def add_in_between(txt_range):
"""Add the simple text found between the last migrated node
and the current node.
Ensures that the text that should remain untransformed
is included in the final file.
"""
if txt_range[0] > last_migrated_char:
try:
original_str = src[last_migrated_char:txt_range[0]]
file_migration.add_string(
StringMigration(
original=original_str,
new=original_str,
confidence=Confidence.HIGH,
))
except Exception as e:
Color.echo(
'[error]Error while adding in-between content '
'in [file]{}[end]. last_migrated_char={}.'
'Error: {}'.format(filename, last_migrated_char,
e))
raise
# Create a map with the text range for each node to migrate
# We need this in order to sort the nodes. This way, we can
# support the migration of imports that appear after function
# calls (e.g. locally)
text_ranges = {}
to_migrate = ([x.node
for x in visitor.imports] + visitor.function_calls)
for node in to_migrate:
text_ranges[node] = attree.get_text_range(node)
# Remove duplicates
to_migrate = sorted(set(to_migrate),
key=lambda n: text_ranges[n][0])
import_added = False
# Create a migration for adding the import statement
# of Native. At this moment we don't know if it will need
# to include t, lazyt or both, but we'll update the instance
# after all nodes have been processed
native_import_string_migration = StringMigration('', '')
native_functions = set() # will store 't'/'lazyt' if found later
for node in to_migrate:
text_range = text_ranges[node]
add_in_between(text_range)
try:
original = src[text_range[0]:text_range[1]]
new = original
confidence = Confidence.HIGH
# Migrate ImportFrom nodes. Leave Import nodes intact,
# as they may have been added in the code for uses other
# than gettext calls
if isinstance(node, ast.ImportFrom):
try:
#
if not import_added:
file_migration.add_string(
native_import_string_migration)
new, item_native_functions = self._transform_import(
visitor, node)
confidence = Confidence.HIGH
import_added = True
native_functions.update(item_native_functions)
# If the whole import statement was about gettext
# functions, a new empty line will have been
# added to the file. In that case,
# this will also remove the empty line completely
if new == '' and \
node.first_token.line == original + '\n':
text_range = (text_range[0], text_range[1] + 1)
original = original + '\n'
except Exception as e:
raise Exception('Error while migrating import'
'\n{}'.format(str(e)))
# Migrate function calls
elif isinstance(node, ast.Call):
try:
new, confidence = self._transform_call(
node, visitor, attree)
except Exception as e:
raise Exception('Error while migrating call'
'\n{}'.format(str(e)))
# If this function call was part of a % operation
# make sure the right part is also removed
modulo_node = visitor.modulos.get(node)
if modulo_node:
# Override the text range with that of the
# modulo operation. This includes the function
# call as well
text_range = attree.get_text_range(modulo_node)
original = src[text_range[0]:text_range[1]]
file_migration.add_string(
StringMigration(
original=original,
new=new,
confidence=confidence,
))
last_migrated_char = text_range[1]
except Exception as e:
Color.echo('[error]Error while transforming content '
'of [file]{}[end]: {}'.format(filename, e))
Color.echo('Original content:\n{}'.format(original))
# Store an exception for this particular file
self.errors.append((filename, e))
return None
# Add the rest of the file (from the last migrated node
# to the end)
if last_migrated_char < len(src):
original = src[last_migrated_char:]
file_migration.add_string(
StringMigration(original=original, new=original))
# Update the Native import statement with the proper functions
if native_functions:
native_import_string_migration.update(
extra_original='',
extra_new=self.import_statement.format(', '.join(
sorted(native_functions))))
return file_migration
def test_revert(self):
migration = StringMigration('original', 'new')
migration.revert()
assert migration.modified is False
assert migration.original == 'original'
assert migration.new == 'original'
def test_constructor_creates_modified_false_string(self):
migration = StringMigration('identical_strings', 'identical_strings')
assert migration.modified is False
assert migration.original == 'identical_strings'
assert migration.new == 'identical_strings'
assert migration.confidence == Confidence.HIGH
def test_constructor_creates_modified_string(self):
migration = StringMigration('original', 'new')
assert migration.modified is True
assert migration.original == 'original'
assert migration.new == 'new'
assert migration.confidence == Confidence.HIGH
def _file_migration():
migration = FileMigration('path/filename.html', 'the content')
migration.add_string(StringMigration('the content',
'the migrated content'))
return migration
