def testGetNextLineContinued(self):
line_1 = "Line part 1."
line_2 = "Line part 2."
line = "%s %s\n%s" % (line_1, CONTINUED_STG, line_2)
extractor = LineExtractor(line)
extractor._getNextLine()
self.assertTrue(line_1 in extractor._current_line)
self.assertTrue(line_2 in extractor._current_line)
def __init__(self, data, mode):
"""
:param data
"""
self.logger = Logger.create(name=__name__)
self.bow_num = BowNum()
self.line_extractor = LineExtractor()
self.word_extractor = WordExtractor()
self.data = data
self.mode = mode
def __init__(self, template_string):
"""
:param str template_string: string containing template variables
and template escape statements to execute
"""
self._extractor = LineExtractor(template_string)
self._message = StatusMessage(self._extractor)
self._executor = Executor()
self._expander = Expander(self._executor, self._message)
self._command = None # Command being processed
self._define_variable_statements = []
def __init__(self, data, mode):
"""
:param data
"""
self.logger = Logger.create(name=__name__)
self.bow_num = BowNum()
self.line_extractor = LineExtractor()
self.word_extractor = WordExtractor()
self.data = data
self.mode = mode
def testGetNextLine(self):
if IGNORE_TEST:
return
extractor = LineExtractor(TEMPLATE_TRAN)
extractor._getNextLine()
lines = []
idx = 0
expecteds = TEMPLATE_TRAN.split('\n')
while extractor._current_line is not None:
expected = expecteds[idx].strip()
idx += 1
lines.append(extractor._current_line)
self.assertEqual(extractor._current_line, expected)
extractor._getNextLine()
expected = len(expecteds)
self.assertEqual(expected, len(lines))
class Patch:
UUID_ADDED, UUID_REMOVED, UUID_MIXED = [
uuid.uuid4().hex[:6] for _ in range(3)
]
@unique
class Mode(Enum):
RESERVED_WORD_ONLY = 1
LINE_TYPE_SENSITIVE = 2
LINE_TYPE_INSENSITIVE = 3
@classmethod
def from_string(cls, s):
try:
return cls[s]
except KeyError:
raise ValueError()
def __init__(self, data, mode):
"""
:param data
"""
self.logger = Logger.create(name=__name__)
self.bow_num = BowNum()
self.line_extractor = LineExtractor()
self.word_extractor = WordExtractor()
self.data = data
self.mode = mode
def normalized(self):
"""
Make sure patches are encoded in unicode: unicode(patch, 'utf-8')
:rtype: unicode
:return:
"""
patches = self.data[:, Column.patch]
self.logger.info('Started extracting patch lines with %r' % self.mode)
if self.mode is self.Mode.LINE_TYPE_SENSITIVE:
self.logger.info(
'(UUID_ADDED, UUID_REMOVED, UUID_MIXED) = (%r, %r, %r)' %
(self.UUID_ADDED, self.UUID_REMOVED, self.UUID_MIXED))
clean_patches = [u''] * len(patches)
invalid_patches = []
for index, patch in enumerate(patches):
try:
clean_patches[index] = self.extract(patch.splitlines())
except UnidiffParseError as e:
# @todo Recover 445 patches at total in vcc_data.npz
invalid_patches.append((index, patch, e))
if len(clean_patches) == len(invalid_patches):
raise AppError('Failed to proceed all patches. Check diff format.')
self.logger.info(
'Completed extracting lines including #%d invalid patches.' %
len(invalid_patches))
return clean_patches
def sensitive(self, lines, suffix):
return [
self.word_extractor.suffix_words(self.bow_num.bin_str(l), suffix)
for l in lines
]
def extract(self, lines):
if self.mode is self.Mode.RESERVED_WORD_ONLY:
return u' '.join([
self.word_extractor.extract_words(l)
for l in self.line_extractor.extract_lines(lines)
])
elif self.mode is self.Mode.LINE_TYPE_INSENSITIVE:
return self.bow_num.bin_str(u' '.join(
self.line_extractor.extract_lines(lines)))
elif self.mode is self.Mode.LINE_TYPE_SENSITIVE:
b = self.sensitive(self.line_extractor.extract_lines(lines),
self.UUID_MIXED)
b.extend(
self.sensitive(self.line_extractor.extract_added_lines(lines),
self.UUID_ADDED))
b.extend(
self.sensitive(
self.line_extractor.extract_removed_lines(lines),
self.UUID_REMOVED))
return u' '.join(b)
else:
raise AppError("Choose mode from %r" % self.Mode.__name__)
class TemplateProcessor(object):
"""
This class processes an Antimony model written using template variable substitutions.
See the project README for syntax details.
"""
def __init__(self, template_string):
"""
:param str template_string: string containing template variables
and template escape statements to execute
"""
self._extractor = LineExtractor(template_string)
self._message = StatusMessage(self._extractor)
self._executor = Executor()
self._expander = Expander(self._executor, self._message)
self._command = None # Command being processed
self._define_variable_statements = []
@classmethod
def processFile(cls, inpath, outpath):
"""
Processes template strings in a file.
:param str inpath: path to the file containing the templated model
:param str outpath: path to the file where the flattened model is placed
"""
template = ''
with open(inpath, 'r') as infile:
for line in infile:
template += "\n" + line
processor = cls(template)
expansion = processor.do()
with open(outpath, 'w') as outfile:
outfile.write(expansion)
@staticmethod
def _makeComment(line):
return "%s%s" % (COMMENT_STG, line)
def _processCommand(self):
"""
Handles command processing, either the current line
is a command or in the midst of processing a paired command.
:param list-of-str expansion:
:return bool: True if processed line
"""
line = self._extractor.getCurrentLine()
line_type = self._extractor.getCurrentLineType()
is_processed = False
# Current line is a command
if line_type == LINE_COMMAND:
is_processed = True
# Check for nested commands
if self._command is not None:
new_command = Command(line)
# Is this a paired command?
if new_command.getCommandVerb() \
== self._command.getCommandVerb():
if new_command.isEnd():
pass
else:
self._message.error("Cannot nest commands")
# Valid placement for a command.
self._command = Command(line)
# DefineVariables Command
if self._command.isDefineVariables():
if self._command.isBegin():
self._define_variables_statements = []
elif self._command.isEnd():
try:
program = '\n'.join(self._define_variables_statements)
self._executor.doScript(program)
except Exception as err:
msg = "***Error %s executing in : \n%s" \
% (str(err), program)
self._message.error(msg)
self._command = None
# SetVersion command
elif self._command.isSetVersion():
version = self._command.getArguments()[0]
if float(version) > float(VERSION):
self._message.error("Unsupported version %s" % version)
self._command = None
# Other commands
else:
self._message.error("Unknown command")
# Process statements occurring within paired commands
elif self._command is not None:
is_processed = True
if self._command.isDefineVariables() and self._command.isBegin():
self._define_variables_statements.append(line)
else:
self._message.error("Invalid paired command.")
return is_processed
def do(self):
"""
Processes the template string and returns the expanded lines for input
to road runner.
Phases
1. Construct content lines (non-blank, not comments)
2. Extract the template variable definitions
3. Construct the substitution instances
4. Process the lines with template variables
State used:
reads: _definitions
:return str expanded_string:
:raises ValueError: errors encountered in the template string
"""
cls = TemplateProcessor
expansion = []
line, line_type = self._extractor.do()
statements = []
while line is not None:
if self._processCommand():
expansion.append(cls._makeComment(line))
# No command being processed
else:
# Transparent line (comment)
if line_type == LINE_TRAN:
expansion.append(line)
elif line_type == LINE_NONE:
pass
# Line to be substituted
elif line_type == LINE_SUBS:
# Do the variable substitutions
try:
substitutions = self._expander.do(line)
except Exception as err:
msg = "Runtime error in expression"
self._message.error(msg)
if len(substitutions) > 1:
expansion.append(cls._makeComment(line))
expansion.extend(substitutions)
else:
import pdb
pdb.set_trace()
raise RuntimeError("Unexepcted state")
line, line_type = self._extractor.do()
if self._command is not None:
msg = "Still processing command %s at EOF" % str(self._command)
self._message.error(msg)
return "\n".join(expansion)
class Patch:
UUID_ADDED, UUID_REMOVED, UUID_MIXED = [uuid.uuid4().hex[:6] for _ in range(3)]
@unique
class Mode(Enum):
RESERVED_WORD_ONLY = 1
LINE_TYPE_SENSITIVE = 2
LINE_TYPE_INSENSITIVE = 3
@classmethod
def from_string(cls, s):
try:
return cls[s]
except KeyError:
raise ValueError()
def __init__(self, data, mode):
"""
:param data
"""
self.logger = Logger.create(name=__name__)
self.bow_num = BowNum()
self.line_extractor = LineExtractor()
self.word_extractor = WordExtractor()
self.data = data
self.mode = mode
def normalized(self):
"""
Make sure patches are encoded in unicode: unicode(patch, 'utf-8')
:rtype: unicode
:return:
"""
patches = self.data[:, Column.patch]
self.logger.info('Started extracting patch lines with %r' % self.mode)
if self.mode is self.Mode.LINE_TYPE_SENSITIVE:
self.logger.info('(UUID_ADDED, UUID_REMOVED, UUID_MIXED) = (%r, %r, %r)'
% (self.UUID_ADDED, self.UUID_REMOVED, self.UUID_MIXED))
clean_patches = [u''] * len(patches)
invalid_patches = []
for index, patch in enumerate(patches):
try:
clean_patches[index] = self.extract(patch.splitlines())
except UnidiffParseError as e:
# @todo Recover 445 patches at total in vcc_data.npz
invalid_patches.append((index, patch, e))
if len(clean_patches) == len(invalid_patches):
raise AppError('Failed to proceed all patches. Check diff format.')
self.logger.info('Completed extracting lines including #%d invalid patches.' % len(invalid_patches))
return clean_patches
def sensitive(self, lines, suffix):
return [
self.word_extractor.suffix_words(self.bow_num.bin_str(l), suffix)
for l in lines
]
def extract(self, lines):
if self.mode is self.Mode.RESERVED_WORD_ONLY:
return u' '.join(
[self.word_extractor.extract_words(l) for l in self.line_extractor.extract_lines(lines)]
)
elif self.mode is self.Mode.LINE_TYPE_INSENSITIVE:
return self.bow_num.bin_str(u' '.join(
self.line_extractor.extract_lines(lines)
))
elif self.mode is self.Mode.LINE_TYPE_SENSITIVE:
b = self.sensitive(self.line_extractor.extract_lines(lines), self.UUID_MIXED)
b.extend(self.sensitive(self.line_extractor.extract_added_lines(lines), self.UUID_ADDED))
b.extend(self.sensitive(self.line_extractor.extract_removed_lines(lines), self.UUID_REMOVED))
return u' '.join(b)
else:
raise AppError("Choose mode from %r" % self.Mode.__name__)
def setUp(self):
self.extractor = LineExtractor("")
self.message = StatusMessage(self.extractor)
self.expander = Expander(Executor(), self.message)
def _testClassifyLine(self, template, expected_classification):
extractor = LineExtractor(template)
extractor._getNextLine()
extractor._classifyLine()
self.assertEqual(extractor.getCurrentLineType(),
expected_classification)
def setUp(self):
self.extractor = LineExtractor(TEMPLATE_TRAN)
def setUp(self):
self.extractor = LineExtractor("")
self.message = StatusMessage(self.extractor)