def remove_trailing_spaces_walker(arguments, directory_name, names):
"""
Walker method to be used by the path walker for the removal of trailing
spaces and newlines.
:type arguments: Tuple
:param arguments: The arguments tuple sent by the walker method.
:type directory_name: String
:param directory_name: The name of the current directory in the walk.
:type names: List
:param names: The list of names in the current directory.
"""
# unpacks the arguments tuple
tab_to_spaces, trailing_newlines, windows_newline,\
file_extensions, file_exclusion = arguments
# tries to run the handle ignore operation for the current set of names and
# in case there's a processing returns the control flow immediately as no
# more handling is meant to occur for the current operation (ignored)
if extra.handle_ignore(names): return
# removes the complete set of names that are meant to be excluded from the
# current set names to be visit (avoid visiting them)
for exclusion in file_exclusion:
if not exclusion in names: continue
names.remove(exclusion)
# retrieves the valid names for the names list (removes directory entries)
valid_complete_names = [
directory_name + "/" + name for name in names\
if not os.path.isdir(directory_name + "/" + name)
]
# filters the names with non valid file extensions
valid_complete_names = [
os.path.normpath(name) for name in valid_complete_names\
if file_extensions == None or os.path.split(name)[-1].split(".")[-1] in file_extensions
]
# creates the string based value of the windows newline taking into
# account the boolean value of it
windows_newline_s = "windows newline" if windows_newline else "unix newline"
# iterates over all the valid complete names with extension filter
for valid_complete_name in valid_complete_names:
# prints a message about the operation to be performed
extra.echo("Removing trail in file: %s (%s)" % (valid_complete_name, windows_newline_s))
# removes the trailing spaces for the (path) name
remove_trailing_spaces(valid_complete_name, tab_to_spaces, windows_newline)
# in case the trailing newlines flag is active
if trailing_newlines:
# prints a message about the operation to be performed
extra.echo("Removing trail newlines in file: %s" % valid_complete_name)
# removes the trailing newlines for the(path) name
remove_trailing_newlines(valid_complete_name, windows_newline)
def cleanup_stylesheets_walker(arguments, directory_name, names):
"""
Walker method to be used by the path walker for the cleanup
of stylesheets.
:type arguments: Tuple
:param arguments: The arguments tuple sent by the walker method.
:type directory_name: String
:param directory_name: The name of the current directory in the walk.
:type names: List
:param names: The list of names in the current directory.
"""
# unpacks the arguments tuple
windows_newline, fix_extra_newlines, property_order,\
rules_skip, file_extensions, file_exclusion = arguments
# tries to run the handle ignore operation for the current set of names and
# in case there's a processing returns the control flow immediately as no
# more handling is meant to occur for the current operation (ignored)
if extra.handle_ignore(names): return
# removes the complete set of names that are meant to be excluded from the
# current set names to be visit (avoid visiting them)
for exclusion in file_exclusion:
if not exclusion in names: continue
names.remove(exclusion)
# retrieves the valid names for the names list (removes directory entries)
valid_complete_names = [
directory_name + "/" + name for name in names\
if not os.path.isdir(directory_name + "/" + name)
]
# filters the names with non valid file extensions so that
valid_complete_names = [
os.path.normpath(name) for name in valid_complete_names\
if file_extensions == None or os.path.split(name)[-1].split(".")[-1] in file_extensions
]
# iterates over all the valid complete names with extension filter
for valid_complete_name in valid_complete_names:
extra.echo("Cleaning stylesheet file: %s" % valid_complete_name)
# removes the cleanups the stylesheet for the (path) name
cleanup_stylesheets(
valid_complete_name,
windows_newline,
fix_extra_newlines,
property_order,
rules_skip
)
def misc_walker(arguments, directory_name, names):
"""
Walker method to be used by the path walker for running the
normalization misc process.
:type arguments: Tuple
:param arguments: The arguments tuple sent by the walker method.
:type directory_name: String
:param directory_name: The name of the current directory in the walk.
:type names: List
:param names: The list of names in the current directory.
"""
# unpacks the arguments tuple into its values
file_exclusion, configuration = arguments
# retrieves the complete set of extensions registered for the chmod
# operations and then gathers their keys as the basis for the extension
# based filtering operations (optimization)
chmod = configuration.get("chmod", {})
extensions = legacy.keys(chmod)
extensions = tuple(extensions)
# tries to run the handle ignore operation for the current set of names and
# in case there's a processing returns the control flow immediately as no
# more handling is meant to occur for the current operation (ignored)
if extra.handle_ignore(names): return
# removes the complete set of names that are meant to be excluded from the
# current set names to be visit (avoid visiting them)
for exclusion in file_exclusion:
if not exclusion in names: continue
names.remove(exclusion)
# retrieves the valid names for the names list (removes directory entries)
valid_complete_names = [directory_name + "/" + name for name in names\
if not os.path.isdir(directory_name + "/" + name)]
# filters the names with non valid file extensions so that only the
# ones that conform with the misc source ones are selected
valid_complete_names = [os.path.normpath(name) for name in valid_complete_names\
if name.endswith(tuple(extensions))]
# iterates over all the valid complete names with valid structure
# as defined by the misc file structure definition
for valid_complete_name in valid_complete_names:
# print a message a message about the misc
# operation that is going to be performed and
# then runs the operation with the correct path
extra.echo("Running the misc operations on file: %s" % valid_complete_name)
misc_file(valid_complete_name, configuration)
def jssource_walker(arguments, directory_name, names):
"""
Walker method to be used by the path walker for running the
normalization jssource process.
:type arguments: Tuple
:param arguments: The arguments tuple sent by the walker method.
:type directory_name: String
:param directory_name: The name of the current directory in the walk.
:type names: List
:param names: The list of names in the current directory.
"""
# unpacks the arguments tuple
file_exclusion, = arguments
# tries to run the handle ignore operation for the current set of names and
# in case there's a processing returns the control flow immediately as no
# more handling is meant to occur for the current operation (ignored)
if extra.handle_ignore(names): return
# removes the complete set of names that are meant to be excluded from the
# current set names to be visit (avoid visiting them)
for exclusion in file_exclusion:
if not exclusion in names: continue
names.remove(exclusion)
# retrieves the valid names for the names list (removes directory entries)
valid_complete_names = [directory_name + "/" + name for name in names\
if not os.path.isdir(directory_name + "/" + name)]
# filters the names with non valid file extensions so that only the
# ones that conform with the javascript source ones are selected
valid_complete_names = [os.path.normpath(name) for name in valid_complete_names\
if name.endswith((".js", ".json"))]
# iterates over all the valid complete names with valid structure
# as defined by the javascript file structure definition
for valid_complete_name in valid_complete_names:
# print a message a message about the jssource
# operation that is going to be performed and
# then runs the operation with the correct path
extra.echo("Transforming javascript source file: %s" % valid_complete_name)
jssource_file(valid_complete_name)
def join_files_walker(arguments, directory_name, names):
"""
Walker method to be used by the path walker for the joining of the files.
:type arguments: Tuple
:param arguments: The arguments tuple sent by the walker method.
:type directory_name: String
:param directory_name: The name of the current directory in the walk.
:type names: List
:param names: The list of names in the current directory.
"""
# unpacks the arguments tuple
file_exclusion, = arguments
# tries to run the handle ignore operation for the current set of names and
# in case there's a processing returns the control flow immediately as no
# more handling is meant to occur for the current operation (ignored)
if extra.handle_ignore(names): return
# removes the complete set of names that are meant to be excluded from the
# current set names to be visit (avoid visiting them)
for exclusion in file_exclusion:
if not exclusion in names: continue
names.remove(exclusion)
# retrieves the valid names for the names list (removes directory entries)
valid_complete_names = [directory_name + "/" + name for name in names\
if not os.path.isdir(directory_name + "/" + name)]
# filters the names with non valid file extensions so that only the
# ones that contain the join extension are used
valid_complete_names = [os.path.normpath(name) for name in valid_complete_names\
if len(name.split(".")) > 1 and name.split(".")[-2] == "join"\
and name.split(".")[-1] == "json"]
# iterates over all the valid complete names with extension filter
for valid_complete_name in valid_complete_names:
# print a message a message about the joining
# operation that is going to be performed and
# then runs the operation with the correct path
extra.echo("Joining files defined in file: %s" % valid_complete_name)
join_files(valid_complete_name)
def run():
# retrieves the path to the "current" directory
directory_path = os.path.dirname(__file__)
# in case the number of arguments is not sufficient
# must print an error message about the issue
if len(sys.argv) < 2:
# prints a series of message about the correct usage
# of the command line for this command
extra.echo("Invalid number of arguments")
extra.echo("Usage: " + USAGE_MESSAGE)
# exits the system in error
sys.exit(2)
# retrieves the target path for execution and the
# extra arguments to be used
target_path = sys.argv[1]
extra_arguments = sys.argv[2:]
# retrieves the current os name and then using it
# sets the shell value to be used in the process
shell_value = os.name in WINDOWS_PLATFORMS and True or False
# sets the initial value for the exit code that is
# going to be used at the end of the cleanup operation
exit_code = 0
info = config(target_path)
# iterates over all the scripts for execution, passing
# the proper script values into each script for execution
for script in info["scripts"]:
# retrieves the script configuration file name in case
# no configuration path is found no value is provided
script_configuration_file_name = info["configuration"].get(script, None)
# creates both the script and the configuration paths
# note that there's a path normalization process
script_path = os.path.join(directory_path, script)
configuration_path = os.path.join(
directory_path,
os.path.join(CONFIGURATION_RELATIVE_PATH, script_configuration_file_name)
) if script_configuration_file_name else None
# resolves both paths as absolute so that the proper value
# is passed to the underlying command line execution
if not script_path == None: script_path = os.path.abspath(script_path)
if not configuration_path == None: configuration_path = os.path.abspath(configuration_path)
# creates the arguments list from the various
# processed arguments
arguments = [info["command"]]
arguments.append(script_path)
arguments.append(target_path)
if configuration_path:
arguments.append(CONFIGURATION_FLAG)
arguments.append(configuration_path)
arguments.extend(extra_arguments)
# prints a message and flushes the standard output
extra.echo("------------------------------------------------------------------------")
extra.echo("Executing script file: %s" % script)
extra.echo("------------------------------------------------------------------------")
sys.stdout.flush()
# opens a sub-process for script execution (and waits for the end of it)
process = subprocess.Popen(
arguments,
stdin = sys.stdin,
stdout = sys.stdout,
stderr = sys.stderr,
shell = shell_value
)
process.wait()
sys.stdout.flush()
# in case the return code of the process is not the expected one (zero)
# the final exit code to be returned by the process is changed to error
if not process.returncode == 0: exit_code = 1
# creates the proper prefix message using the return code from the process
# note that the failed prefix is set for unwanted error codes
prefix = "Finished" if process.returncode == 0 else "Finished (with errors)"
# print a message and flushes the standard output
extra.echo("------------------------------------------------------------------------")
extra.echo("%s executing script file: %s" % (prefix, script))
extra.echo("------------------------------------------------------------------------")
sys.stdout.flush()
# exits the current process with the code that has been calculated
# from the execution of the various sub-scripts
sys.exit(exit_code)
def main():
"""
Main function used for the joining files.
"""
# in case the number of arguments
# is not sufficient
if len(sys.argv) < 2:
# prints a series of message related with he
# correct usage of the command line and then
# exits the process with error indication
extra.echo("Invalid number of arguments")
extra.echo("Usage: " + USAGE_MESSAGE)
sys.exit(2)
# sets the default values for the parameters
path = sys.argv[1]
recursive = False
file_exclusion = None
configuration_file_path = None
try:
options, _arguments = getopt.getopt(sys.argv[2:], "rw:c:", [])
except getopt.GetoptError:
# prints a series of messages about the
# correct usage of the command line and
# exits the current process with an error
extra.echo("Invalid number of arguments")
extra.echo("Usage: " + USAGE_MESSAGE)
sys.exit(2)
# iterates over all the options, retrieving the option
# and the value for each
for option, value in options:
if option == "-r":
recursive = True
elif option == "-w":
file_exclusion = [value.strip() for value in value.split(",")]
elif option == "-c":
configuration_file_path = value
# retrieves the configurations from the command line arguments
# either from the command line or configuration file
configurations = extra.configuration(
file_path = configuration_file_path,
recursive = recursive,
file_exclusion = file_exclusion
)
# iterates over all the configurations, executing them
for configuration in configurations:
# retrieves the configuration values
recursive = configuration["recursive"]
file_exclusion = configuration["file_exclusion"]
# in case the recursive flag is set, joins the files in
# recursive mode (multiple files)
if recursive: join_files_recursive(path, file_exclusion)
# otherwise it's a "normal" iteration and joins the
# files (for only one file)
else: join_files(path)
# verifies if there were messages printed to the standard
# error output and if that's the case exits in error
sys.exit(1 if extra.has_errors() else 0)
def main():
"""
Main function used for the processing of the stylesheet
files so that a normalized form is outputted.
"""
# in case the number of arguments
# is not sufficient
if len(sys.argv) < 2:
# prints a series of message about the correct
# usage of the command line and the exits the
# current process with an error code
extra.echo("Invalid number of arguments")
extra.echo("Usage: " + USAGE_MESSAGE)
sys.exit(2)
# sets the default values for the parameters
path = sys.argv[1]
recursive = False
windows_newline = True
fix_extra_newlines = False
property_order = None
rules_skip = None
file_extensions = None
file_exclusion = None
configuration_file_path = None
try:
options, _arguments = getopt.getopt(sys.argv[2:], "rp:e:w:c:", [])
except getopt.GetoptError:
# prints a series of message about the correct
# usage of the command line and the exits the
# current process with an error code
extra.echo("Invalid number of arguments")
extra.echo("Usage: " + USAGE_MESSAGE)
sys.exit(2)
# iterates over all the options, retrieving the option
# and the value for each
for option, value in options:
if option == "-r":
recursive = True
if option == "-u":
windows_newline = True
if option == "-n":
fix_extra_newlines = True
if option == "-p":
property_order = [value.strip() for value in value.split(",")]
if option == "-s":
rules_skip = [value.strip() for value in value.split(",")]
elif option == "-e":
file_extensions = [value.strip() for value in value.split(",")]
elif option == "-w":
file_exclusion = [value.strip() for value in value.split(",")]
elif option == "-c":
configuration_file_path = value
# retrieves the configurations from the command line arguments
configurations = extra.configuration(
file_path = configuration_file_path,
recursive = recursive,
windows_newline = windows_newline,
fix_extra_newlines = fix_extra_newlines,
property_order = property_order,
rules_skip = rules_skip,
file_extensions = file_extensions,
file_exclusion = file_exclusion
)
# iterates over all the configurations, executing them
for configuration in configurations:
# retrieves the configuration values
recursive = configuration["recursive"]
windows_newline = configuration["windows_newline"]
fix_extra_newlines = configuration["fix_extra_newlines"]
property_order = configuration["property_order"] or ()
rules_skip = configuration["rules_skip"] or ()
file_extensions = configuration["file_extensions"] or ()
file_exclusion = configuration["file_exclusion"] or ()
# in case the recursive flag is set, removes the trailing
# spaces in recursive mode
if recursive:
cleanup_stylesheets_recursive(
path,
windows_newline,
fix_extra_newlines,
property_order,
rules_skip,
file_extensions,
file_exclusion
)
# otherwise it's a "normal" iteration, removes the trailing
# spaces (for only one file)
else:
cleanup_stylesheets(
path,
windows_newline,
fix_extra_newlines,
property_order,
rules_skip
)
# verifies if there were messages printed to the standard
# error output and if that's the case exits in error
sys.exit(1 if extra.has_errors() else 0)
def main():
"""
Main function used for the encoding conversion.
"""
# in case the number of arguments
# is not sufficient
if len(sys.argv) < 2:
# prints a series of messages about the command line
# error that occurred and then exits with an error
extra.echo("Invalid number of arguments")
extra.echo("Usage: " + USAGE_MESSAGE)
sys.exit(2)
# sets the default values for the parameters
path = sys.argv[1]
recursive = False
source_encoding = None
target_encoding = None
windows_newline = None
replacements_list = None
file_extensions = None
file_exclusion = None
configuration_file_path = None
try:
options, _arguments = getopt.getopt(sys.argv[2:], "rs:t:x:e:w:c:", [])
except getopt.GetoptError:
# prints a series of messages about the command line
# error that occurred and then exits with an error
extra.echo("Invalid number of arguments")
extra.echo("Usage: " + USAGE_MESSAGE)
sys.exit(2)
# iterates over all the options, retrieving the option
# and the value for each
for option, value in options:
if option == "-r":
recursive = True
elif option == "-s":
source_encoding = value
elif option == "-t":
target_encoding = value
elif option == "-u":
windows_newline = value
elif option == "-x":
replacements_list = [value.strip() for value in value.split(",")]
elif option == "-e":
file_extensions = [value.strip() for value in value.split(",")]
elif option == "-w":
file_exclusion = [value.strip() for value in value.split(",")]
elif option == "-c":
configuration_file_path = value
# retrieves the configurations from the command line arguments
configurations = extra.configuration(
file_path = configuration_file_path,
recursive = recursive,
source_encoding = source_encoding,
target_encoding = target_encoding,
windows_newline = windows_newline,
replacements_list = replacements_list,
file_extensions = file_extensions,
file_exclusion = file_exclusion
)
# iterates over all the configurations, executing them
for configuration in configurations:
# retrieves the configuration values
recursive = configuration["recursive"]
source_encoding = configuration["source_encoding"] or "utf-8"
target_encoding = configuration["target_encoding"] or "utf-8"
windows_newline = configuration["windows_newline"] or True
replacements_list = configuration["replacements_list"] or ()
file_extensions = configuration["file_extensions"] or ()
file_exclusion = configuration["file_exclusion"] or ()
# in case the recursive flag is set must converts the files
# using the recursive mode
if recursive:
convert_encoding_recursive(
path,
source_encoding,
target_encoding,
windows_newline,
replacements_list,
file_extensions,
file_exclusion
)
# otherwise it's a "normal" iteration, must converts the
# encoding (for only one file)
else:
convert_encoding(
path,
source_encoding,
target_encoding,
windows_newline,
replacements_list
)
# verifies if there were messages printed to the standard
# error output and if that's the case exits in error
sys.exit(1 if extra.has_errors() else 0)
def convert_encoding_walker(arguments, directory_name, names):
"""
Walker method to be used by the path walker for the encoding conversion.
:type arguments: Tuple
:param arguments: The arguments tuple sent by the walker method.
:type directory_name: String
:param directory_name: The name of the current directory in the walk.
:type names: List
:param names: The list of names in the current directory.
"""
# unpacks the arguments tuple
source_encoding, target_encoding, windows_newline,\
replacements_list, file_extensions, file_exclusion = arguments
# tries to run the handle ignore operation for the current set of names and
# in case there's a processing returns the control flow immediately as no
# more handling is meant to occur for the current operation (ignored)
if extra.handle_ignore(names): return
# removes the complete set of names that are meant to be excluded from the
# current set names to be visit (avoid visiting them)
for exclusion in file_exclusion:
if not exclusion in names: continue
names.remove(exclusion)
# retrieves the valid names for the names list (removes directory entries)
valid_complete_names = [directory_name + "/" + name\
for name in names if not os.path.isdir(directory_name + "/" + name)]
# filters the names with non valid file extensions
valid_complete_names = [os.path.normpath(name) for name in valid_complete_names\
if file_extensions == None or os.path.split(name)[-1].split(".")[-1] in file_extensions]
# creates the string based value of the windows newline taking into
# account the boolean value of it
windows_newline_s = "windows newline" if windows_newline else "unix newline"
# iterates over all the valid complete names with extension filter
# to convert the respective file into the target encoding
for valid_complete_name in valid_complete_names:
# prints a message about the file that is not going to be converted
# into the proper target encoding as defined in the specification
extra.echo(
"Convert encoding in file: %s (%s to %s) (%s)" %\
(
valid_complete_name,
source_encoding,
target_encoding,
windows_newline_s
)
)
try:
# converts the encoding for the provided (path) name according to
# a set of defined options, for various reasons this operation may
# fail if such thing happens the operation is skipped
convert_encoding(
valid_complete_name,
source_encoding,
target_encoding,
windows_newline,
replacements_list
)
except Exception:
extra.warn(
"Failed converting encoding in file: %s (%s to %s)" %\
(
valid_complete_name,
source_encoding,
target_encoding
)
)
def main():
"""
Main function used for the removal of both the trailing spaces
and the extra newlines.
"""
# in case the number of arguments
# is not sufficient
if len(sys.argv) < 2:
# prints a series of message about the error
# that has just occurred and then exits with
# an error code to the calling process
extra.echo("Invalid number of arguments")
extra.echo("Usage: " + USAGE_MESSAGE)
sys.exit(2)
# sets the default values for the parameters
path = sys.argv[1]
recursive = False
tab_to_spaces = False
trailing_newlines = False
windows_newline = True
file_extensions = None
file_exclusion = None
configuration_file_path = None
try:
options, _arguments = getopt.getopt(sys.argv[2:], "rtnue:w:c:", [])
except getopt.GetoptError:
# prints a series of message about the error
# that has just occurred and then exits with
# an error code to the calling process
extra.echo("Invalid number of arguments")
extra.echo("Usage: " + USAGE_MESSAGE)
sys.exit(2)
# iterates over all the options, retrieving the option
# and the value for each
for option, value in options:
if option == "-r":
recursive = True
elif option == "-t":
tab_to_spaces = True
elif option == "-n":
trailing_newlines = True
elif option == "-u":
windows_newline = False
elif option == "-e":
file_extensions = [value.strip() for value in value.split(",")]
elif option == "-w":
file_exclusion = [value.strip() for value in value.split(",")]
elif option == "-c":
configuration_file_path = value
# retrieves the configurations from the command line arguments
configurations = extra.configuration(
file_path = configuration_file_path,
recursive = recursive,
tab_to_spaces = tab_to_spaces,
trailing_newlines = trailing_newlines,
windows_newline = windows_newline,
file_extensions = file_extensions,
file_exclusion = file_exclusion
)
# iterates over all the configurations, executing them
for configuration in configurations:
# retrieves the configuration values
recursive = configuration["recursive"]
tab_to_spaces = configuration["tab_to_spaces"]
trailing_newlines = configuration["trailing_newlines"]
windows_newline = configuration["windows_newline"]
file_extensions = configuration["file_extensions"] or ()
file_exclusion = configuration["file_exclusion"] or ()
# in case the recursive flag is set
if recursive:
# removes the trailing spaces in recursive mode
remove_trailing_spaces_recursive(
path,
tab_to_spaces,
trailing_newlines,
windows_newline,
file_extensions,
file_exclusion
)
# otherwise it's a "normal" iteration
else:
# removes the trailing spaces (for one file)
remove_trailing_spaces(path, tab_to_spaces, windows_newline)
# in case the trailing newlines
if trailing_newlines:
# removes the trailing newlines (for one file)
remove_trailing_newlines(path, windows_newline)
# verifies if there were messages printed to the standard
# error output and if that's the case exits in error
sys.exit(1 if extra.has_errors() else 0)