def __search_circular_inheritance(ModeDB, mode, inheritance_path):
def __report_error(mode, inheritance_path):
assert inheritance_path != []
msg = ""
previous = mode.name
inheritance_path.reverse()
for mode_name in inheritance_path:
msg += "mode '%s' is a base of mode '%s'.\n" % (previous, mode_name)
previous = mode_name
error_msg("circular inheritance detected:\n" + msg, mode.filename, mode.line_n)
for base_mode_name in mode.base_modes:
# -- does base mode exist?
if ModeDB.has_key(base_mode_name) == False:
error_msg("mode '%s' is derived from mode a '%s'\n" % (mode.name, base_mode_name) + \
"but no such mode '%s' actually exists!" % base_mode_name,
mode.filename, mode.line_n)
# -- did mode appear in the path of deriving modes
base_mode = ModeDB[base_mode_name]
if base_mode.name in inheritance_path: __report_error(base_mode, inheritance_path)
__search_circular_inheritance(ModeDB, base_mode, inheritance_path + [mode.name])
def add_match_deletion(self, Pattern, PatternStateMachine, FileName, LineN):
if self.__matches.has_key(Pattern):
error_msg("Deletion of '%s' which appeared before in same mode.\n" % Pattern + \
"Deletion of pattern.", FileName, LineN)
PatternStateMachine = transformation.do(PatternStateMachine)
self.__deletion_db[Pattern] = [PatternStateMachine, FileName, LineN]
def do(ARGV):
"""Performs a query based on the given command line arguments.
RETURNS: True if a query was performed.
False if not query was requested.
"""
cl = GetPot(ARGV, SectionsEnabledF=False)
success_f = False
# Regular Expressions extract the BufferLimitCode and the PathTerminatorCode
# from the sets. So let us define them outside the normal range.
backup_buffer_limit_code = Setup.buffer_limit_code
backup_path_limit_code = Setup.path_limit_code
Setup.buffer_limit_code = -1
Setup.path_limit_code = -1
try:
success_f = True
if search_and_validate(cl, "--codec-info"): __handle_codec(cl)
elif search_and_validate(cl, "--codec-for-language"): __handle_codec_for_language(cl)
elif search_and_validate(cl, "--property"): __handle_property(cl)
elif search_and_validate(cl, "--set-by-property"): __handle_set_by_property(cl)
elif search_and_validate(cl, "--set-by-expression"): __handle_set_by_expression(cl)
elif search_and_validate(cl, "--property-match"): __handle_property_match(cl)
else: success_f = False
except RegularExpressionException, x:
error_msg(x.message)
def add_match_priority(self, Pattern, PatternStateMachine, PatternIdx, FileName, LineN):
if self.__matches.has_key(Pattern):
error_msg("Pattern '%s' appeared twice in mode definition.\n" % Pattern + \
"Only this priority mark is considered.", FileName, LineN)
PatternStateMachine = transformation.do(PatternStateMachine)
self.__repriorization_db[Pattern] = [PatternStateMachine, FileName, LineN, PatternIdx]
def __add_new_pattern_action_pair(pattern_action_pair_list,
PatternActionPair):
state_machine = PatternActionPair.pattern_state_machine()
pattern = PatternActionPair.pattern
for earlier_match in pattern_action_pair_list:
if earlier_match.pattern_state_machine().get_id(
) > state_machine.get_id():
continue
if subset_checker.do(earlier_match.pattern_state_machine(),
state_machine) == False:
continue
file_name, line_n = earlier_match.get_action_location()
error_msg(
"Pattern '%s' matches a superset of what is matched by" %
earlier_match.pattern,
file_name,
line_n,
DontExitF=True,
WarningF=False)
file_name, line_n = PatternActionPair.get_action_location()
error_msg("pattern '%s' while the former has precedence.\n" % \
pattern + "The latter can never match.\n" + \
"You may switch the sequence of definition to avoid this error.",
file_name, line_n)
# Here we already quit by error_msg(...).
else:
# Shallow copy is enough! Later on, there might be actions that
# generate source code, and then the source code takes the place of
# the action. For this to work, inherited actions must be de-antangled.
pattern_action_pair_list.append(copy(PatternActionPair))
def seal(self):
if len(self.space_db) == 0 and len(self.grid_db) == 0:
default_space = ord(' ')
default_tab = ord('\t')
bad = self.bad_character_set
if bad.get().contains(default_space) == False:
self.specify_space("[ ]", NumberSet(default_space), 1, self.fh)
if bad.get().contains(default_tab) == False:
self.specify_grid("[\\t]", NumberSet(default_tab), 4, self.fh)
if len(self.space_db) == 0 and len(self.grid_db) == 0:
error_msg(
"No space or grid defined for indentation counting. Default\n"
"values ' ' and '\\t' could not be used since they are specified as 'bad'.",
bad.file_name, bad.line_n)
if self.newline_state_machine.get() == None:
sm = StateMachine()
end_idx = sm.add_transition(sm.init_state_index,
NumberSet(ord('\n')),
AcceptanceF=True)
mid_idx = sm.add_transition(sm.init_state_index,
NumberSet(ord('\r')),
AcceptanceF=False)
sm.add_transition(mid_idx,
NumberSet(ord('\n')),
end_idx,
AcceptanceF=False)
self.specify_newline("(\\r\\n)|(\\n)", sm, self.fh)
def __check_on_orphan_states(Place, sm):
orphan_state_list = sm.get_orphaned_state_index_list()
if orphan_state_list == []: return
error_msg("After '%s'" % Place + "\n" + \
"Orphaned state(s) detected in regular expression (optimization lack).\n" + \
"Please, log a defect at the projects website quex.sourceforge.net.\n" + \
"Orphan state(s) = " + repr(orphan_state_list) + "\n")
def __error_character_set_intersection(Before):
error_msg("Character set specification '%s' intersects" % Name,
FH,
DontExitF=True,
WarningF=False)
error_msg("with definition for '%s' at this place." % Before.name,
Before.file_name, Before.line_n)
def __start_mode(applicable_mode_name_list, mode_name_list):
"""If more then one mode is defined, then that requires an explicit
definition 'start = mode'.
"""
assert len(applicable_mode_name_list) != 0
start_mode = lexer_mode.initial_mode.get_pure_code()
if start_mode == "":
# Choose an applicable mode as start mode
start_mode = applicable_mode_name_list[0]
lexer_mode.initial_mode = CodeFragment(start_mode)
if len(applicable_mode_name_list) > 1:
error_msg("No initial mode defined via 'start' while more than one applicable mode exists.\n" + \
"Use for example 'start = %s;' in the quex source file to define an initial mode." \
% start_mode)
# This Branch: start mode is applicable and present
else:
FileName = lexer_mode.initial_mode.filename
LineN = lexer_mode.initial_mode.line_n
# Start mode present and applicable?
verify_word_in_list(start_mode, mode_name_list,
"Start mode '%s' is not defined." % start_mode,
FileName, LineN)
verify_word_in_list(start_mode, applicable_mode_name_list,
"Start mode '%s' is inheritable only and cannot be instantiated." % start_mode,
FileName, LineN)
def __start_mode(applicable_mode_name_list, mode_name_list):
"""If more then one mode is defined, then that requires an explicit
definition 'start = mode'.
"""
assert len(applicable_mode_name_list) != 0
start_mode = lexer_mode.initial_mode.get_pure_code()
if start_mode == "":
# Choose an applicable mode as start mode
start_mode = applicable_mode_name_list[0]
lexer_mode.initial_mode = CodeFragment(start_mode)
if len(applicable_mode_name_list) > 1:
error_msg("No initial mode defined via 'start' while more than one applicable mode exists.\n" + \
"Use for example 'start = %s;' in the quex source file to define an initial mode." \
% start_mode)
# This Branch: start mode is applicable and present
else:
FileName = lexer_mode.initial_mode.filename
LineN = lexer_mode.initial_mode.line_n
# Start mode present and applicable?
verify_word_in_list(start_mode, mode_name_list,
"Start mode '%s' is not defined." % start_mode,
FileName, LineN)
verify_word_in_list(
start_mode, applicable_mode_name_list,
"Start mode '%s' is inheritable only and cannot be instantiated." %
start_mode, FileName, LineN)
def __search_circular_inheritance(ModeDB, mode, inheritance_path):
def __report_error(mode, inheritance_path):
assert inheritance_path != []
msg = ""
previous = mode.name
inheritance_path.reverse()
for mode_name in inheritance_path:
msg += "mode '%s' is a base of mode '%s'.\n" % (previous,
mode_name)
previous = mode_name
error_msg("circular inheritance detected:\n" + msg, mode.filename,
mode.line_n)
for base_mode_name in mode.base_modes:
# -- does base mode exist?
if ModeDB.has_key(base_mode_name) == False:
error_msg("mode '%s' is derived from mode a '%s'\n" % (mode.name, base_mode_name) + \
"but no such mode '%s' actually exists!" % base_mode_name,
mode.filename, mode.line_n)
# -- did mode appear in the path of deriving modes
base_mode = ModeDB[base_mode_name]
if base_mode.name in inheritance_path:
__report_error(base_mode, inheritance_path)
__search_circular_inheritance(ModeDB, base_mode,
inheritance_path + [mode.name])
def __check_file_name(setup, Candidate, Name):
value = setup.__dict__[Candidate]
CommandLineOption = SETUP_INFO[Candidate][0]
if value == "": return
if type(value) == list:
for name in value:
if name != "" and name[0] == "-":
error_msg("Quex refuses to work with file names that start with '-' (minus).\n" + \
"Received '%s' for %s (%s)" % (value, name, repr(CommandLineOption)[1:-1]))
if os.access(name, os.F_OK) == False:
# error_msg("File %s (%s)\ncannot be found." % (name, Name))
error_msg_file_not_found(name, Name)
else:
if value == "" or value[0] == "-": return
if os.access(value, os.F_OK): return
if os.access(QUEX_PATH + "/" + value, os.F_OK): return
if os.access(os.path.dirname(value), os.F_OK) == False \
and os.access(QUEX_PATH + "/" + os.path.dirname(value), os.F_OK) == False:
error_msg("File '%s' is supposed to be located in directory '%s' or\n" % \
(os.path.basename(value), os.path.dirname(value)) + \
"'%s'. No such directories exist." % \
(QUEX_PATH + "/" + os.path.dirname(value)))
error_msg_file_not_found(value, Name)
def __post_process(fh, StartPosition, object, ReturnRE_StringF):
assert object == None \
or isinstance(object, StateMachine) \
or isinstance(object, NumberSet)
if isinstance(fh, StringIO):
regular_expression = ""
else:
end_position = fh.tell()
fh.seek(StartPosition)
regular_expression = fh.read(end_position - StartPosition)
if regular_expression == "":
regular_expression = fh.read(1)
fh.seek(-1, 1)
# (*) error in regular expression?
if object == None:
error_msg(
"No valid regular expression detected, found '%s'." %
regular_expression, fh)
# NOT: Do not transform here, since transformation might happen twice when patterns
# are defined and when they are replaced.
if ReturnRE_StringF: return regular_expression, object
else: return object
def __check_file_name(setup, Candidate, Name):
value = setup.__dict__[Candidate]
CommandLineOption = SETUP_INFO[Candidate][0]
if value == "": return
if type(value) == list:
for name in value:
if name != "" and name[0] == "-":
error_msg("Quex refuses to work with file names that start with '-' (minus).\n" + \
"Received '%s' for %s (%s)" % (value, name, repr(CommandLineOption)[1:-1]))
if os.access(name, os.F_OK) == False:
# error_msg("File %s (%s)\ncannot be found." % (name, Name))
error_msg_file_not_found(name, Name)
else:
if value == "" or value[0] == "-": return
if os.access(value, os.F_OK): return
if os.access(QUEX_PATH + "/" + value, os.F_OK): return
if os.access(os.path.dirname(value), os.F_OK) == False \
and os.access(QUEX_PATH + "/" + os.path.dirname(value), os.F_OK) == False:
error_msg("File '%s' is supposed to be located in directory '%s' or\n" % \
(os.path.basename(value), os.path.dirname(value)) + \
"'%s'. No such directories exist." % \
(QUEX_PATH + "/" + os.path.dirname(value)))
error_msg_file_not_found(value, Name)
def __check_on_orphan_states(Place, sm):
orphan_state_list = sm.get_orphaned_state_index_list()
if orphan_state_list == []: return
error_msg("After '%s'" % Place + "\n" + \
"Orphaned state(s) detected in regular expression (optimization lack).\n" + \
"Please, log a defect at the projects website quex.sourceforge.net.\n" + \
"Orphan state(s) = " + repr(orphan_state_list) + "\n")
def parse_character_set(Txt_or_File, PatternStringF=False):
if Txt_or_File.__class__ in [file, StringIO]:
sh = Txt_or_File
sh_ref = sh
position = sh.tell()
else:
sh = StringIO(Txt_or_File)
sh_ref = -1
start_position = sh.tell()
try:
# -- parse regular expression, build state machine
character_set = charset_expression.snap_set_expression(sh)
if character_set == None:
error_msg("No valid regular character set expression detected.", sh_ref)
# -- character set is not supposed to contain buffer limit code
if character_set.contains(Setup.buffer_limit_code):
character_set.cut_interval(Interval(Setup.buffer_limit_code, Setup.buffer_limit_code))
except RegularExpressionException, x:
error_msg("Regular expression parsing:\n" + x.message, sh_ref)
def do(ARGV):
"""Performs a query based on the given command line arguments.
RETURNS: True if a query was performed.
False if not query was requested.
"""
cl = GetPot(ARGV, SectionsEnabledF=False)
success_f = False
# Regular Expressions extract the BufferLimitCode and the PathTerminatorCode
# from the sets. So let us define them outside the normal range.
backup_buffer_limit_code = Setup.buffer_limit_code
backup_path_limit_code = Setup.path_limit_code
Setup.buffer_limit_code = -1
Setup.path_limit_code = -1
try:
success_f = True
if search_and_validate(cl, "--codec-info"): __handle_codec(cl)
elif search_and_validate(cl, "--codec-for-language"):
__handle_codec_for_language(cl)
elif search_and_validate(cl, "--property"):
__handle_property(cl)
elif search_and_validate(cl, "--set-by-property"):
__handle_set_by_property(cl)
elif search_and_validate(cl, "--set-by-expression"):
__handle_set_by_expression(cl)
elif search_and_validate(cl, "--property-match"):
__handle_property_match(cl)
else:
success_f = False
except RegularExpressionException, x:
error_msg(x.message)
def __handle_property_match(cl):
property_follower = cl.follow("", "--property-match")
sys.stderr.write("(please, wait for database parsing to complete)\n")
if property_follower == "":
return
fields = map(lambda x: x.strip(), property_follower.split("="))
if len(fields) != 2:
error_msg("Wrong property setting '%s'." % result)
# -- determine name and value
name = fields[0]
wild_card_expression = fields[1]
# -- get the property from the database
property = __get_property(name)
if property == None:
return True
# -- find the character set for the given expression
if property.type == "Binary":
error_msg("Binary property '%s' is not subject to value wild card matching.\n" % property.name)
for value in property.get_wildcard_value_matches(wild_card_expression):
print value
def __handle_property_match(cl):
property_follower = cl.follow("", "--property-match")
sys.stderr.write("(please, wait for database parsing to complete)\n")
if property_follower == "":
return
fields = map(lambda x: x.strip(), property_follower.split("="))
if len(fields) != 2:
error_msg("Wrong property setting '%s'." % result)
# -- determine name and value
name = fields[0]
wild_card_expression = fields[1]
# -- get the property from the database
property = __get_property(name)
if property == None:
return True
# -- find the character set for the given expression
if property.type == "Binary":
error_msg(
"Binary property '%s' is not subject to value wild card matching.\n"
% property.name)
for value in property.get_wildcard_value_matches(wild_card_expression):
print value
def set(self, Value, fh):
if self.__value != None:
error_msg("%s has been defined more than once.\n" % self.name, fh, DontExitF=True)
error_msg("previous definition has been here.\n", self.file_name, self.line_n)
self.__value = Value
self.file_name = fh.name
self.line_n = get_current_line_info_number(fh)
def __get_float(MemberName):
ValueStr = setup.__dict__[MemberName]
if type(ValueStr) == float: return ValueStr
try:
return float(ValueStr)
except:
option_name = repr(SETUP_INFO[MemberName][0])[1:-1]
error_msg("Cannot convert '%s' into an floating point number for '%s'" % (ValueStr, option_name))
def __check_identifier(setup, Candidate, Name):
value = setup.__dict__[Candidate]
if is_identifier(value): return
CommandLineOption = SETUP_INFO[Candidate][0]
error_msg("%s must be a valid identifier (%s).\n" % (Name, repr(CommandLineOption)[1:-1]) + \
"Received: '%s'" % value)
def add_match_deletion(self, Pattern, PatternStateMachine, FileName,
LineN):
if self.__matches.has_key(Pattern):
error_msg("Deletion of '%s' which appeared before in same mode.\n" % Pattern + \
"Deletion of pattern.", FileName, LineN)
PatternStateMachine = transformation.do(PatternStateMachine)
self.__deletion_db[Pattern] = [PatternStateMachine, FileName, LineN]
def __check_identifier(setup, Candidate, Name):
value = setup.__dict__[Candidate]
if is_identifier(value): return
CommandLineOption = SETUP_INFO[Candidate][0]
error_msg("%s must be a valid identifier (%s).\n" % (Name, repr(CommandLineOption)[1:-1]) + \
"Received: '%s'" % value)
def __validate_marks(DB, DoneDB, CommentStr):
ok_f = True
for pattern, info in DB.items():
if DoneDB.has_key(pattern): continue
ok_f = False
error_msg("Pattern '%s' was marked %s but does not\n" % (pattern, CommentStr) + \
"exist in any base mode of mode '%s'." % self.name,
info[1], info[2], DontExitF=True, WarningF=False)
return ok_f
def __validate_marks(DB, DoneDB, CommentStr):
ok_f = True
for pattern, info in DB.items():
if DoneDB.has_key(pattern): continue
ok_f = False
error_msg("Pattern '%s' was marked %s but does not\n" % (pattern, CommentStr) + \
"exist in any base mode of mode '%s'." % self.name,
info[1], info[2], DontExitF=True, WarningF=False)
return ok_f
def __get_integer(code, option_name):
try:
if type(code) == int: return code
elif len(code) > 2:
if code[:2] == "0x": return int(code, 16)
elif code[:2] == "0o": return int(code, 8)
return int(code)
except:
error_msg("Cannot convert '%s' into an integer for '%s'" % (code, option_name))
def get_supported_graphic_formats():
reply_str = _call_dot("", "?", "", GetStdErrF=True)
list_start_i = reply_str.rfind(":")
if list_start_i == -1 or list_start_i == len(reply_str)-1:
error_msg("Graphviz was asked to report supported graphic formats, but\n" + \
"reply was incomprehensible.")
return reply_str[list_start_i+1:].split()
def __handle_deletion_and_repriorization(CurrentModeName, pattern_action_pair_list,
repriorization_db, deletion_db):
def __validate_marks(DB, DoneDB, CommentStr):
ok_f = True
for pattern, info in DB.items():
if DoneDB.has_key(pattern): continue
ok_f = False
error_msg("Pattern '%s' was marked %s but does not\n" % (pattern, CommentStr) + \
"exist in any base mode of mode '%s'." % self.name,
info[1], info[2], DontExitF=True, WarningF=False)
return ok_f
def __is_in_patterns(AllegedIdenticalSM, MyDB):
for pattern, info in MyDB.items():
sm = info[0]
if identity_checker.do(AllegedIdenticalSM, sm): return pattern
return ""
# DELETION / PRIORITY-MARK
deletion_done_db = {}
repriorization_done_db = {}
i = 0
size = len(pattern_action_pair_list)
while i < size:
match = pattern_action_pair_list[i]
state_machine = match.pattern_state_machine()
found_pattern = __is_in_patterns(state_machine, deletion_db)
if found_pattern != "":
# Delete pattern from the list of pattern action pairs
del pattern_action_pair_list[i]
size -= 1
# Mark 'deletion applied'
deletion_done_db[found_pattern] = True
self.__history_deletion.append([CurrentModeName, match.pattern, match.mode_name])
continue
found_pattern = __is_in_patterns(state_machine, repriorization_db)
if found_pattern != "":
# Adapt the pattern index, this automatically adapts the match precedence
old_state_machine_id = state_machine.get_id()
new_state_machine_id = repriorization_db[found_pattern][-1]
new_match = deepcopy(match)
new_match.pattern_state_machine().core().set_id(new_state_machine_id)
pattern_action_pair_list[i] = new_match
# Mark 'repriorization applied'
repriorization_done_db[found_pattern] = True
self.__history_repriorization.append([CurrentModeName, match.pattern, match.mode_name,
old_state_machine_id, new_state_machine_id])
i += 1
# Ensure that all mentioned marks really had some effect.
if not __validate_marks(deletion_db, deletion_done_db, "for DELETION") \
or not __validate_marks(repriorization_db, repriorization_done_db, "with PRIORITY-MARK"):
error_msg("Abort.")
return
def __get_integer(code, option_name):
try:
if type(code) == int: return code
elif len(code) > 2:
if code[:2] == "0x": return int(code, 16)
elif code[:2] == "0o": return int(code, 8)
return int(code)
except:
error_msg("Cannot convert '%s' into an integer for '%s'" %
(code, option_name))
def __report_error(mode, inheritance_path):
assert inheritance_path != []
msg = ""
previous = mode.name
inheritance_path.reverse()
for mode_name in inheritance_path:
msg += "mode '%s' is a base of mode '%s'.\n" % (previous, mode_name)
previous = mode_name
error_msg("circular inheritance detected:\n" + msg, mode.filename, mode.line_n)
def search_and_validate(CL, Option):
if CL.search(Option) == False: return False
# Validate command line
ufos = CL.unidentified_options(OPTION_DB.keys())
if ufos != []:
error_msg("Unidentified option(s) = " + repr(ufos) + "\n" + \
get_supported_command_line_option_description())
return True
def search_and_validate(CL, Option):
if CL.search(Option) == False: return False
# Validate command line
ufos = CL.unidentified_options(OPTION_DB.keys())
if ufos != []:
error_msg("Unidentified option(s) = " + repr(ufos) + "\n" + \
get_supported_command_line_option_description())
return True
def add_match_priority(self, Pattern, PatternStateMachine, PatternIdx,
FileName, LineN):
if self.__matches.has_key(Pattern):
error_msg("Pattern '%s' appeared twice in mode definition.\n" % Pattern + \
"Only this priority mark is considered.", FileName, LineN)
PatternStateMachine = transformation.do(PatternStateMachine)
self.__repriorization_db[Pattern] = [
PatternStateMachine, FileName, LineN, PatternIdx
]
def __get_float(MemberName):
ValueStr = setup.__dict__[MemberName]
if type(ValueStr) == float: return ValueStr
try:
return float(ValueStr)
except:
option_name = repr(SETUP_INFO[MemberName][0])[1:-1]
error_msg(
"Cannot convert '%s' into an floating point number for '%s'" %
(ValueStr, option_name))
def __codec_vs_buffer_element_size(CodecName, RequiredBufferElementSize):
if setup.buffer_codec != CodecName: return
elif setup.buffer_element_size == RequiredBufferElementSize: return
if setup.buffer_element_size == -1:
msg_str = "undetermined (found type '%s')" % setup.buffer_element_type
else:
msg_str = "is not %i (found %i)" % (RequiredBufferElementSize, setup.buffer_element_size)
error_msg("Using codec '%s' while buffer element size %s.\n" % (CodecName, msg_str) +
"Consult command line argument '--buffer-element-size'.")
def set(self, Value, fh):
if self.__value != None:
error_msg("%s has been defined more than once.\n" % self.name,
fh,
DontExitF=True)
error_msg("previous definition has been here.\n", self.file_name,
self.line_n)
self.__value = Value
self.file_name = fh.name
self.line_n = get_current_line_info_number(fh)
def __check_on_init_state_not_acceptance(Place, sm):
init_state = sm.get_init_state()
if init_state.core().is_acceptance():
error_msg("After '%s'" % Place + "\n" + \
"The initial state is 'acceptance'. This should never appear.\n" + \
"Please, log a defect at the projects website quex.sourceforge.net.\n")
if filter(lambda origin: origin.is_acceptance(), init_state.origins().get_list()) != []:
error_msg("After '%s'" % Place + "\n" + \
"Initial state contains an origin that is 'acceptance'. This should never appear.\n" + \
"Please, log a defect at the projects website quex.sourceforge.net.\n")
def __check_on_init_state_not_acceptance(Place, sm):
init_state = sm.get_init_state()
if init_state.core().is_acceptance():
error_msg("After '%s'" % Place + "\n" + \
"The initial state is 'acceptance'. This should never appear.\n" + \
"Please, log a defect at the projects website quex.sourceforge.net.\n")
if filter(lambda origin: origin.is_acceptance(),
init_state.origins().get_list()) != []:
error_msg("After '%s'" % Place + "\n" + \
"Initial state contains an origin that is 'acceptance'. This should never appear.\n" + \
"Please, log a defect at the projects website quex.sourceforge.net.\n")
def __report_error(mode, inheritance_path):
assert inheritance_path != []
msg = ""
previous = mode.name
inheritance_path.reverse()
for mode_name in inheritance_path:
msg += "mode '%s' is a base of mode '%s'.\n" % (previous,
mode_name)
previous = mode_name
error_msg("circular inheritance detected:\n" + msg, mode.filename,
mode.line_n)
def __get_integer(MemberName):
ValueStr = setup.__dict__[MemberName]
if type(ValueStr) == int: return ValueStr
result = read_integer(StringIO(ValueStr))
if result == None:
option_name = repr(SETUP_INFO[MemberName][0])[1:-1]
error_msg("Cannot convert '%s' into an integer for '%s'.\n" % (ValueStr, option_name) + \
"Use prefix '0x' for hexadecimal numbers.\n" + \
" '0o' for octal numbers.\n" + \
" '0b' for binary numbers.\n" + \
" '0r' for roman numbers.\n" + \
" and no prefix for decimal numbers.")
return result
def parse(fh, AllowNothingIsFineF=False):
start_position = fh.tell()
try:
# (*) parse regular expression, build state machine
pattern_state_machine = regex.do(fh, lexer_mode.shorthand_db,
DOS_CarriageReturnNewlineF = Setup.dos_carriage_return_newline_f,
AllowNothingIsNecessaryF = AllowNothingIsFineF)
except RegularExpressionException, x:
fh.seek(start_position)
error_msg("Regular expression parsing:\n" + x.message, fh)
def __get_integer(MemberName):
ValueStr = setup.__dict__[MemberName]
if type(ValueStr) == int: return ValueStr
result = read_integer(StringIO(ValueStr))
if result == None:
option_name = repr(SETUP_INFO[MemberName][0])[1:-1]
error_msg("Cannot convert '%s' into an integer for '%s'.\n" % (ValueStr, option_name) + \
"Use prefix '0x' for hexadecimal numbers.\n" + \
" '0o' for octal numbers.\n" + \
" '0b' for binary numbers.\n" + \
" '0r' for roman numbers.\n" + \
" and no prefix for decimal numbers.")
return result
def __codec_vs_buffer_element_size(CodecName, RequiredBufferElementSize):
if setup.buffer_codec != CodecName: return
elif setup.buffer_element_size == RequiredBufferElementSize: return
if setup.buffer_element_size == -1:
msg_str = "undetermined (found type '%s')" % setup.buffer_element_type
else:
msg_str = "is not %i (found %i)" % (RequiredBufferElementSize,
setup.buffer_element_size)
error_msg("Using codec '%s' while buffer element size %s.\n" %
(CodecName, msg_str) +
"Consult command line argument '--buffer-element-size'.")
def __get_mode_db(Setup):
# (0) check basic assumptions
if Setup.input_mode_files == []: error_msg("No input files.")
# (1) input: do the pattern analysis, in case exact counting of newlines is required
# (this might speed up the lexer, but nobody might care ...)
# pattern_db = analyse_patterns.do(pattern_file)
mode_db = quex_file_parser.do(Setup.input_mode_files, Setup)
# (*) perform consistency check
consistency_check.do(mode_db)
return mode_db
def do(Modes):
"""If consistency check fails due to a fatal error, then this functions
exits back to the system with error code -1. Otherwise, in 'not so
perfect' cases there might be only a warning being printed to standard
output.
"""
if len(Modes) == 0:
error_msg("No single mode defined - bailing out",
Prefix="consistency check")
# -- is there a mode that is applicable?
for mode in Modes.values():
if mode.options["inheritable"] != "only": break
else:
error_msg("There is no mode that can be applied---all existing modes are 'inheritable only'.\n" + \
"modes are = " + repr(map(lambda m: m.name, Modes.values()))[1:-1],
Prefix="consistency check")
# -- is the initial mode defined
if lexer_mode.initial_mode.get_code() == "":
# find first mode that can actually be applied
for mode in Modes.values():
if mode.options["inheritable"] != "only":
selected_mode = mode.name
break
lexer_mode.initial_mode = CodeFragment(selected_mode)
error_msg("no initial mode defined via 'start'\n" + \
"using mode '%s' as initial mode" % selected_mode, DontExitF=True,
Prefix="warning")
# -- is the start mode applicable?
if Modes.has_key(lexer_mode.initial_mode.get_code()) == False:
error_msg(
"Start mode '%s' has not been defined anywhere." %
lexer_mode.initial_mode.get_code(),
lexer_mode.initial_mode.filename, lexer_mode.initial_mode.line_n)
if Modes[lexer_mode.initial_mode.get_code(
)].options["inheritable"] == "only":
error_msg(
"Start mode '%s' is inheritable only and cannot be instantiated." %
lexer_mode.initial_mode.get_code(),
lexer_mode.initial_mode.filename, lexer_mode.initial_mode.line_n)
# -- check for circular inheritance
check_circular_inheritance(Modes)
# -- mode specific checks
for mode in Modes.values():
mode.consistency_check()
def parse(fh, AllowNothingIsFineF=False):
start_position = fh.tell()
try:
# (*) parse regular expression, build state machine
pattern_state_machine = regex.do(
fh,
lexer_mode.shorthand_db,
DOS_CarriageReturnNewlineF=Setup.dos_carriage_return_newline_f,
AllowNothingIsNecessaryF=AllowNothingIsFineF)
except RegularExpressionException, x:
fh.seek(start_position)
error_msg("Regular expression parsing:\n" + x.message, fh)
def parse_character_string(Txt_or_File, PatternStringF=False):
sh, sh_ref, start_position = __prepare_text_or_file_stream(Txt_or_File)
try:
# -- parse regular expression, build state machine
state_machine = snap_character_string.do(sh)
state_machine = regex.__clean_and_validate(state_machine, AllowNothingIsNecessaryF=False, fh=sh)
if state_machine == None:
error_msg("No valid regular character string expression detected.", sh_ref)
except RegularExpressionException, x:
error_msg("Regular expression parsing:\n" + x.message, sh_ref)
def specify_grid(self, PatternStr, CharSet, Count, FH=-1):
self.__check("grid", self.grid_db, CharSet, FH, Key=Count)
if Count == 0:
error_msg(
"A grid count of 0 is nonsense. May be define a space count of 0.",
FH)
if Count == 1:
error_msg("Indentation grid counts of '1' are equivalent of to a space\n" + \
"count of '1'. The latter is faster to compute.",
FH, DontExitF=True)
self.grid_db[Count] = LocalizedParameter("grid", CharSet, FH)
self.grid_db[Count].pattern_str = PatternStr
def specify_grid(self, PatternStr, CharSet, Count, FH=-1):
self.__check("grid", self.grid_db, CharSet, FH, Key=Count)
if Count == 0:
error_msg("A grid count of 0 is nonsense. May be define a space count of 0.", FH)
if Count == 1:
error_msg(
"Indentation grid counts of '1' are equivalent of to a space\n"
+ "count of '1'. The latter is faster to compute.",
FH,
DontExitF=True,
)
self.grid_db[Count] = LocalizedParameter("grid", CharSet, FH)
self.grid_db[Count].pattern_str = PatternStr
def __init__(self, PatternActionPairList, StateMachineName, GraphicFormat):
assert map(lambda elm: elm.__class__.__name__, PatternActionPairList) \
== [ "PatternActionInfo" ] * len(PatternActionPairList)
assert_graphviz_installed()
GeneratorBase.__init__(self, PatternActionPairList, StateMachineName, SupportBeginOfLineF=False)
# -- check if graphics format is supported
supported_format_list = get_supported_graphic_formats()
if GraphicFormat not in supported_format_list:
error_msg("Graphic format '%s' not supported.\n" % GraphicFormat + \
get_supported_graphic_format_description())
self.graphic_format = GraphicFormat
def __commonality(mode, Info, ReferenceSM, Name):
for pattern_action_pair in mode.get_pattern_action_pair_list():
if pattern_action_pair.comment in ["indentation newline", "indentation newline suppressor"]:
continue
sm = pattern_action_pair.pattern_state_machine()
if commonality_checker.do(ReferenceSM, sm) != commonality_checker.NONE:
error_msg("The %s pattern '%s'" \
% (Name, Info.pattern_str), Info.file_name, Info.line_n,
DontExitF=True, WarningF=False)
pattern = pattern_action_pair.pattern
file_name, line_n = pattern_action_pair.get_action_location()
error_msg("has commonalities with pattern '%s' defined here." % pattern,
file_name, line_n)
def __determine_base_mode_sequence(self, ModeDescr, InheritancePath):
"""Determine the sequence of base modes. The type of sequencing determines
also the pattern precedence. The 'deep first' scheme is chosen here. For
example a mode hierarchie of
A
/ \
B C
/ \ / \
D E F G
results in a sequence: (A, B, D, E, C, F, G).reverse()
This means, that patterns and event handlers of 'E' have precedence over
'C' because they are the childs of a preceding base mode.
This function detects circular inheritance.
"""
if ModeDescr.name in InheritancePath:
msg = "mode '%s'\n" % InheritancePath[0]
for mode_name in InheritancePath[InheritancePath.index(ModeDescr.
name) + 1:]:
msg += " inherits mode '%s'\n" % mode_name
msg += " inherits mode '%s'" % ModeDescr.name
error_msg("circular inheritance detected:\n" + msg,
ModeDescr.filename, ModeDescr.line_n)
base_mode_name_list_reversed = deepcopy(ModeDescr.base_modes)
#base_mode_name_list_reversed.reverse()
for name in base_mode_name_list_reversed:
# -- does mode exist?
verify_word_in_list(
name, mode_description_db.keys(),
"Mode '%s' inherits mode '%s' which does not exist." %
(ModeDescr.name, name), ModeDescr.filename, ModeDescr.line_n)
if name in map(lambda m: m.name, self.__base_mode_sequence):
continue
# -- grab the mode description
mode_descr = mode_description_db[name]
self.__determine_base_mode_sequence(
mode_descr, InheritancePath + [ModeDescr.name])
self.__base_mode_sequence.append(ModeDescr)
return self.__base_mode_sequence
def __handle_codec_for_language(cl):
language_name = cl.follow("", "--codec-for-language")
supported_language_list = codec_db.get_supported_language_list()
if language_name == "":
txt = "Missing argument after '--codec-for-language'. Supported languages are:\n\n"
line_txt = ""
for name in supported_language_list:
line_txt += name + ", "
if len(line_txt) > 50: txt += line_txt + "\n"; line_txt = ""
txt += line_txt
txt = txt[:-2] + "."
error_msg(txt)
print "Possible Codecs: " + repr(codec_db.get_codecs_for_language(language_name))[1:-1]
def unicode_to_utf8(UnicodeValue):
if UnicodeValue < 0x80:
return [ UnicodeValue, ]
elif UnicodeValue < 0x800:
return [ 0xC0 | (UnicodeValue >> 6),
0x80 | (UnicodeValue & 0x3F)]
elif UnicodeValue < 0x10000:
return [ 0xE0 | (UnicodeValue >> 12),
0x80 | ((UnicodeValue & 0xFFF) >> 6),
0x80 | (UnicodeValue & 0x3F)]
elif UnicodeValue < 0x00200000:
return [ 0xF0 | (UnicodeValue >> 18),
0x80 | ((UnicodeValue & 0x3FFFF) >> 12),
0x80 | ((UnicodeValue & 0xFFF) >> 6),
0x80 | (UnicodeValue & 0x3F)]
else:
error_msg("Unicode character > 0x10FFFF detected. Cannot be handled.")
def do(Modes):
"""If consistency check fails due to a fatal error, then this functions
exits back to the system with error code -1. Otherwise, in 'not so
perfect' cases there might be only a warning being printed to standard
output.
"""
if len(Modes) == 0:
error_msg("No single mode defined - bailing out", Prefix="consistency check")
# -- is there a mode that is applicable?
for mode in Modes.values():
if mode.options["inheritable"] != "only": break
else:
error_msg("There is no mode that can be applied---all existing modes are 'inheritable only'.\n" + \
"modes are = " + repr(map(lambda m: m.name, Modes.values()))[1:-1],
Prefix="consistency check")
# -- is the initial mode defined
if lexer_mode.initial_mode.get_code() == "":
# find first mode that can actually be applied
for mode in Modes.values():
if mode.options["inheritable"] != "only":
selected_mode = mode.name
break
lexer_mode.initial_mode = CodeFragment(selected_mode)
error_msg("no initial mode defined via 'start'\n" + \
"using mode '%s' as initial mode" % selected_mode, DontExitF=True,
Prefix="warning")
# -- is the start mode applicable?
if Modes.has_key(lexer_mode.initial_mode.get_code()) == False:
error_msg("Start mode '%s' has not been defined anywhere." % lexer_mode.initial_mode.get_code(),
lexer_mode.initial_mode.filename, lexer_mode.initial_mode.line_n)
if Modes[lexer_mode.initial_mode.get_code()].options["inheritable"] == "only":
error_msg("Start mode '%s' is inheritable only and cannot be instantiated." % lexer_mode.initial_mode.get_code(),
lexer_mode.initial_mode.filename, lexer_mode.initial_mode.line_n)
# -- check for circular inheritance
check_circular_inheritance(Modes)
# -- mode specific checks
for mode in Modes.values():
mode.consistency_check()