def process_color_definition(property_line, line_number):
# tries to match the color regex and retrieves the resulting
# set of groups from the match, in case there's no matching
# the control flow is returned immediately to the caller
line_match = COLOR_REGEX.match(property_line)
line_groups = line_match.groups() if line_match else None
if not line_groups: return property_line
# unpacks the various line groups into the appropriate
# variables to be used in the processing of the color
property_name, pre_color, color, post_color = line_groups
try:
# runs the fixing color operation and then, re-joins the
# various groups again into a single string (from list)
color = fix_color(color)
line_groups = [property_name, pre_color, color, post_color]
property_line = "".join(line_groups)
except Exception as exception:
# converts the exception to string and then
# prints a warning to the proper output
exception_string = legacy.UNICODE(exception)
extra.warn("%s near line %d" % (exception_string, line_number))
# returns the property line
return property_line
def get_property_index(property_line, property_order, line_number):
"""
Retrieves the index for the property specified in the provided property
line, this index may be used to correctly position the property.
:type property_line: String
:param property_line: The property line containing the property.
:type property_order: List
:param property_order: The list of property names in order.
:type line_number: int
:param line_number: The approximate line number for this processing.
:rtype: int
:return: The index of the property line, note that in case an index
value is not found for the rule a large index is returned instead.
"""
# splits the property name, to retrieve the property value
# this first splits tries to find also the number of values
# present in the current line
property_line_splitted = property_line.split(":")
# in case the property line did not correctly split must
# returns immediately with the provided property order
if len(property_line_splitted) < 2: return len(property_order)
# runs a second split operation that limits the number of splits
# in the line to two so that no extra problems are raised and then
# retrieves the name of the current property
property_line_splitted = property_line.split(":", 1)
property_name, property_value = property_line_splitted
property_name = property_name.strip()
property_value = property_value.strip()
# in case the property name or value are empty, raises an exception indicating
# the problem to be handled at the top layers
if not property_name: raise Exception("property name is empty")
if not property_value: raise Exception("property value is empty")
# in case the property is not in the order
if not property_name in property_order:
# warns about the missing property name and returns the
# largest possible index value for any property name
extra.warn(
"Order for property %s not defined at line %d" %\
(property_name, line_number)
)
return len(property_order)
# in case the property value starts with a separator it's considered
# that the value is a extra/duplicated separator
if property_value[0] == ":":
# warns about the extra values in the line, so that the duplicated
# information may be properly removed and considered
extra.warn("Extra values found at line %d" % line_number)
# determines the index for the property name and returns
# the value to the caller method
property_index = property_order.index(property_name)
return property_index
def process_property_line(property_line, line_number, indentation):
# in case the property line is empty, when stripped the property
# is considered to be empty (warning required) logs the message
# and then returns the line itself (no processing)
if not property_line.strip():
extra.warn("Empty stylesheet property at line %s" % line_number)
return property_line
# in case no separator is present between the name of the property
# the value of it a warning is raised and the value returned immediately
# to avoid further (unavoidable) errors
if not ":" in property_line:
extra.warn("No value set for property at line %s" % line_number)
return property_line
# strips the line to the right so that no newline characters
# exist (much simpler to manage lines without newlines)
property_line = property_line.rstrip()
property_line = property_line.rstrip(";")
# runs the initial process of simplifying the property by running
# the various rules for the current property name
property_line = process_rules(property_line, line_number)
# ensures the property line is correctly indented and
# the property name and value are correctly separated
padding = indentation * " " * 4
property_line = PROPERTY_LINE_REGEX.sub(
padding + PROPERTY_LINE_REPLACEMENT_VALUE,
property_line
)
# ensures the property line and an ending semicolon
# adding it to the property line in case it does not
# exists (for processing)
is_valid = property_line.endswith(";")
if not is_valid: property_line += ";"
# replaces the urls so that no double quotes are used in
# them and instead the value is used directly
property_line = URL_REGEX.sub(URL_REPLACEMENT_VALUE, property_line)
# processes the color definitions, so that the complete
# color value is used in the processing
property_line = process_color_definition(property_line, line_number)
# returns the processed property line
return property_line
def check_duplicates(property_lines, line_number):
# retrieves the complete set of names for the rules, this is
# going to be used as the base unit for duplicate checking
names = [line.strip().split(":", 1)[0] for line in property_lines]
# starts the sequence/list that is going to store the already
# visited sequence of names (for duplicate detection)
visited = []
# iterates over the complete set of names to detect any duplicated
# value printing a warning message for such situation
for name in names:
if name in visited: extra.warn(
"Duplicated property line '%s' at line %s" %\
(name, line_number)
)
visited.append(name)
def cleanup_properties(
input_buffer,
windows_newline = True,
fix_extra_newlines = True,
property_order = (),
rules_skip = ()
):
"""
Cleans up the property lines. Sorts the css properties in the file,
by the specified property order.
Ensures that all property lines are correctly terminated with semicolon.
:type input_buffer: StringBuffer
:param input_buffer: The input buffer.
:type windows_newline: bool
:param windows_newline: If the windows newline should be used.
:type windows_newline: bool
:param windows_newline: If the extra newlines should be fixed.
:type fix_extra_newlines: bool
:param fix_extra_newlines: If the extra newlines should be fixed.
:type property_order: List
:param property_order: The list with the explicit order of property names.
:type rules_skip: List
:param rules_skip: The list of specific rules to skip.
"""
# reads the input lines and counts the number of lines
# in the buffer setting it as the number of "original" lines
lines = input_buffer.readlines()
number_original_lines = len(lines)
# creates a string buffer that will hold the output of the
# cleanup operation over the current stylesheet file
output_buffer = legacy.StringIO()
# initializes a series of state variables that will control
# the way the parser/generator will work through the file
rule_started = False
line_number = 0
open_rule_count = 0
newlines = 0
comments_started = 0
needs_newline = False
# for each of the input lines it's going to run the iteration
# loop and match it against the proper rules
for line in lines:
# decodes the current line as a processing using unicode
# characters is required for correct results
line = line.decode("utf-8")
# increments the line number as one more line is going
# to be processes through the current iteration
line_number += 1
# updates the comparison key function
get_comparison_key = lambda property_line: get_property_index(
property_line,
property_order,
line_number
)
# in case the line contains a single line comment,
# does nothing, will write the line as is
if "/*" in line and "*/" in line and not "*/*/" in line:
pass
# in case the line contains the end of multiline comment
elif "*/" in line:
# in case the comment mode is currently not enabled prints
# a warning as we're trying to close a comment without opening
if not comments_started: extra.error(
"Found closing comment without corresponding opening at line %d" % line_number
)
# in case there's more that one closing comment defined under
# the same line prints an error as this is not allowed
if line.count("*/") > 1: extra.error(
"More that one closing comment at line %d" % line_number
)
# decrements the comments started counter and then
# enables the needs newline flag
comments_started = 0
needs_newline = True
# in case the line contains a the start of multiline comment
elif "/*" in line:
# in case the comment mode is already enabled (at least one)
# prints a waning about the double opening
if comments_started: extra.warn(
"Found opening comment inside open comment at line %d" % line_number
)
# in case there's more that one opening comment defined under
# the same line prints an error as this is not allowed
if line.count("/*") > 1: extra.error(
"More that one opening comment at line %d" % line_number
)
# increments the comments started counter
comments_started += 1
# in case this is a comment line (comment started)
# does nothing, will just write the line as is
elif comments_started:
pass
# in case the line contains an at rule specification
elif AT_RULES_REGEX.match(line):
# after an at rule, a newline must follow
needs_newline = True
# in case the line contains a full rule specification
# does nothing, will just write line as is as there's
# very few thing possible to optimize under this situation
elif "{" in line and "}" in line:
line = line.strip()
needs_newline = True
rule_started = False
# in case this is a rule start line
elif "{" in line:
# increments the open rule count
open_rule_count += 1
# in case this is an actual rule, meaning that the open rule
# count value is positive (assertion valid)
if open_rule_count > 0:
# resets the rule lines
rule_lines = []
# signals the rule started flag,
# in case the rule is no to be skipped
rule_started = not skip_rule(line, rules_skip)
# replaces the various css tokens with proper separators
# meaning that no extra spaces between them are allowed
line = TOKEN_REGEX.sub(r" \1 ", line)
# runs the space replacement operation so that the
# extra spaces in the line (not required) are removed
# and the line is simplified
line = SPACE_REGEX.sub(" ", line)
line = line.strip()
# calculates the space padding that is going to be pre-pended
# to the line and then adds it to the line note that the open
# rule count is decremented by one (opened on this line)
padding = (open_rule_count - 1) * 4 * " "
line = padding + line
# flushes the current line value to the output buffer and then
# continues the current loop (avoids general operations)
write_line(output_buffer, line, windows_newline)
continue
elif "}" in line:
# decrements the open rule count
open_rule_count -= 1
# in case there is a mismatch in open and closed rules
# must raise an exception indicating the problem
if open_rule_count < 0: raise Exception("mismatched rules found")
# strips the current line removing any extra (not expected)
# values and simplifying the current line value
padding = open_rule_count * 4 * " "
line = line.strip()
line = padding + line
# in case the rule set does not contain any property
# must log an information message about the empty rule
if rule_started and not rule_lines:
extra.warn("Empty stylesheet rule at line %d" % line_number)
# updates the flag to signal the rule has ended
rule_started = False
# sorts the various property lines and the processes them
property_lines = sorted(rule_lines, key = get_comparison_key)
property_lines = process_property_lines(
property_lines,
line_number,
indentation = open_rule_count + 1,
avoid_empty = fix_extra_newlines
)
property_lines = sorted(property_lines, key = get_comparison_key)
# writes the lines to the buffer, considering the windows newline
# and then writes the line
write_lines(
output_buffer,
property_lines,
windows_newline = windows_newline,
avoid_empty = fix_extra_newlines
)
write_line(output_buffer, line, windows_newline = windows_newline)
# resets the newlines counter and then
# enables the needs newline flag
newlines = 0
needs_newline = True
rule_lines = []
# skips further processing
continue
# in case this is part of rule selector declaration
# the typical rule replacement operations are applied
elif line.rstrip().endswith(","):
line = TOKEN_REGEX.sub(r" \1 ", line)
line = SPACE_REGEX.sub(" ", line)
line = COMMA_REGEX.sub(r"\1", line)
line = line.strip()
# in case this line is part of a valid rule set
elif rule_started:
# appends the line to the rule set, and then
# skips outputting the line to the buffer
rule_lines.append(line)
continue
# in case the between rules mode is active
elif not rule_started and not line.strip():
# increments the newlines count
newlines += 1
# otherwise in case this is an extra newline, must either
# remove it or print a warning message depending on mode
if not needs_newline and newlines > 1:
if fix_extra_newlines: continue
else: extra.warn("Found extra newline at line %d" % line_number)
# disables the needs newline flag
needs_newline = False
else:
# warns about the statement outside a valid rule
extra.warn("Found statement outside rule at line %d" % line_number)
# "calculates" the padding string value taking into account the
# current open rule count value and then removes any left space
# compatible values and replaces them with proper indentation
padding = open_rule_count * 4 * " "
line = line.lstrip()
line = padding + line
# writes the line to the output buffer taking into
# account the windows newline control flag
write_line(output_buffer, line, windows_newline)
# in case there is a mismatch in open and closed rules
# must raise an exception indicating the problem
if not open_rule_count == 0: raise Exception("mismatched rules found")
# retrieves the output buffer value and then counts the
# number of lines contained in it (assertion validation)
output_buffer_value = output_buffer.getvalue()
number_lines = output_buffer_value.count("\n")
if not number_lines == number_original_lines and not fix_extra_newlines:
raise Exception(
"number of lines in processed file (%d) is different from original file (%d)" %\
(number_lines, number_original_lines)
)
return output_buffer
def convert_encoding_walker(arguments, directory_name, names):
"""
Walker method to be used by the path walker for the encoding conversion.
:type arguments: Tuple
:param arguments: The arguments tuple sent by the walker method.
:type directory_name: String
:param directory_name: The name of the current directory in the walk.
:type names: List
:param names: The list of names in the current directory.
"""
# unpacks the arguments tuple
source_encoding, target_encoding, windows_newline,\
replacements_list, file_extensions, file_exclusion = arguments
# tries to run the handle ignore operation for the current set of names and
# in case there's a processing returns the control flow immediately as no
# more handling is meant to occur for the current operation (ignored)
if extra.handle_ignore(names): return
# removes the complete set of names that are meant to be excluded from the
# current set names to be visit (avoid visiting them)
for exclusion in file_exclusion:
if not exclusion in names: continue
names.remove(exclusion)
# retrieves the valid names for the names list (removes directory entries)
valid_complete_names = [directory_name + "/" + name\
for name in names if not os.path.isdir(directory_name + "/" + name)]
# filters the names with non valid file extensions
valid_complete_names = [os.path.normpath(name) for name in valid_complete_names\
if file_extensions == None or os.path.split(name)[-1].split(".")[-1] in file_extensions]
# creates the string based value of the windows newline taking into
# account the boolean value of it
windows_newline_s = "windows newline" if windows_newline else "unix newline"
# iterates over all the valid complete names with extension filter
# to convert the respective file into the target encoding
for valid_complete_name in valid_complete_names:
# prints a message about the file that is not going to be converted
# into the proper target encoding as defined in the specification
extra.echo(
"Convert encoding in file: %s (%s to %s) (%s)" %\
(
valid_complete_name,
source_encoding,
target_encoding,
windows_newline_s
)
)
try:
# converts the encoding for the provided (path) name according to
# a set of defined options, for various reasons this operation may
# fail if such thing happens the operation is skipped
convert_encoding(
valid_complete_name,
source_encoding,
target_encoding,
windows_newline,
replacements_list
)
except Exception:
extra.warn(
"Failed converting encoding in file: %s (%s to %s)" %\
(
valid_complete_name,
source_encoding,
target_encoding
)
)
def pydev_file(file_path, fix = True):
"""
Runs the pydev configuration file normalization that consists
in the definition in order of each of the xml lines.
This operation should fail with an exception in case the
structure of the xml document is not the expected one.
:type file_path: String
:param file_path: The path to the file that contains the
pydev configuration specification in xml.
:type fix: bool
:param fix: If any "fixable" error in the pydev project
file should be automatically fixes using the known heuristics,
this is a dangerous option as errors may be created.
"""
paths = []
properties = dict()
buffer = []
xmldoc = xml.dom.minidom.parse(file_path)
nodes = xmldoc.getElementsByTagName("pydev_property")
for node in nodes:
value = text_value(node)
name = node.attributes["name"].value
properties[name] = value
nodes = xmldoc.getElementsByTagName("pydev_pathproperty")
nodes = nodes[0].childNodes if nodes else []
for node in nodes:
value = text_value(node)
if not value: continue
paths.append(value)
for key in legacy.keys(properties):
if key in VALID_PROPERTIES: continue
raise RuntimeError("Invalid property '%s'" % key)
if fix: paths, properties = fix_values(paths, properties)
python_version = properties.get("org.python.pydev.PYTHON_PROJECT_VERSION", None)
if not python_version: extra.warn("No python version defined")
elif not python_version == "python 2.6": extra.warn("Python version not 2.6")
for path in paths:
if path.startswith("/${PROJECT_DIR_NAME}"): continue
extra.warn("Project directory path not normalized '%s'" % path)
property_keys = legacy.keys(properties)
property_keys.sort()
buffer.append("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n")
buffer.append("<?eclipse-pydev version=\"1.0\"?><pydev_project>\n")
if paths: buffer.append("<pydev_pathproperty name=\"org.python.pydev.PROJECT_SOURCE_PATH\">\n")
for path in paths:
buffer.append("<path>%s</path>\n" % path)
if paths: buffer.append("</pydev_pathproperty>\n")
for key in property_keys:
value = properties[key]
buffer.append("<pydev_property name=\"%s\">%s</pydev_property>\n" % (key, value))
buffer.append("</pydev_project>\n")
result = "".join(buffer)
result = result.encode("utf-8")
file = open(file_path, "wb")
try: file.write(result)
finally: file.close()