def remove_trailing_spaces(file_path, tab_to_spaces, windows_newline = True):
"""
Removes the extra spaces or tabs in every line of the
file contents.
The extra argument defines if the newline format used
should be the windows mode (carriage return and newline).
:type file_path: String
:param file_path: The path to the file to have the trailing
spaces removed.
:type tab_to_spaces: bool
:param tab_to_spaces: If the tab characters should be converted
to spaces.
:type windows_newline: bool
:param windows_newline: If the windows newline should be used.
"""
# normalizes the file path
file_path_normalized = extra.normalize_path(file_path)
# opens the file for reading
file = open(file_path_normalized, "rb")
try:
# creates a string buffer for buffering
string_buffer = legacy.BytesIO()
# iterates over all the lines in the file
for line in file:
# strips the line
line_stripped = line.rstrip()
# in case the tab must be replaced with spaces
# performs the operations with the proper way
if tab_to_spaces:
line_stripped = line_stripped.replace(b"\t", SPACE_TAB)
# writes the stripped line to the string buffer
string_buffer.write(line_stripped)
# writes the proper line ending sequence taking into
# account if the windows newline should be used or not
if windows_newline: string_buffer.write(b"\r\n")
else: string_buffer.write(b"\n")
finally:
# closes the file for reading
file.close()
# retrieves the string value from the string buffer
# this should be a bytes based buffer string
string_value = string_buffer.getvalue()
# opens the file for writing and writes the complete
# set of generated contents into it (output operation)
file = open(file_path_normalized, "wb")
try: file.write(string_value)
finally: file.close()
def join_files(file_path):
"""
Runs the joining operation according to the specification
provided by the file located at the provided path.
The joining operation may be complex and may take some
time until it's completely finished.
:type file_path: String
:param file_path: The path to the file that contains the
json based specification for the joining operation.
"""
# normalizes the file path value so that it
# represents a correct file path
file_path_normalized = extra.normalize_path(file_path)
# opens the file for reading and then reads the
# complete set of data contained in it
file = open(file_path_normalized, "rb")
try: file_contents = file.read()
finally: file.close()
# uses the default encoding for json to decode
# the complete set of contents in the file
file_contents = file_contents.decode("utf-8")
# loads the file contents, retrieving the
# map of files to be created from joining
files_map = json.loads(file_contents)
# retrieves the directory path from the normalized
# file path (for context resolution)
directory_path = os.path.dirname(file_path_normalized)
# tries to retrieve the target directories
target_directories = files_map.get("_target_directories", None)
# in case the target directories are
# successfully retrieved
if target_directories:
# removes the target directories from
# the files map (avoids collision)
del files_map["_target_directories"]
# otherwise the target directories to be
# used are the default ones
else:
# sets the default target directories
target_directories = (".",)
# iterates over all the files (composition of joined files)
# from the files map
for file_key, file_value in files_map.items():
# creates the string buffer for temporary
# file holding, this is bytes based
string_buffer = legacy.BytesIO()
# retrieves the current value attributes
# (setting default values)
files = file_value.get("files", [])
minify = file_value.get("minify", None)
compress = file_value.get("compress", None)
# sets the is first flag
is_first = True
# iterates over all the files,
# to be joined
for file in files:
# in case it's the first file
# no need to write the separator
if is_first: is_first = False
# otherwise the separator must
# be written
else: string_buffer.write(b"\r\n")
# retrieves the complete file path by joining the
# directory path and the current "file"
file_path = os.path.join(directory_path, file)
file_path = os.path.abspath(file_path)
# in case the file does not exists, raises an
# error indicating that there was an error
if not os.path.exists(file_path):
raise RuntimeError("the file path does not exist for file '%s'" % file)
# opens the current file for reading
# in binary format and reads the complete
# set of contents into the current buffer
_file = open(file_path, "rb")
try: file_contents = _file.read()
finally: _file.close()
# writes the file contents into the string
# buffer, appending the contents to the same file
string_buffer.write(file_contents)
# retrieves the string value from the buffer, this is
# a binary (byte based) string and should be used with
# the proper care to avoid unwanted results
string_value = string_buffer.getvalue()
# minifies and compresses the string value according
# to the provided specification, some of this operations
# may use complex third-party code
string_value = minify == "javascript" and extra.javascript_minify(string_value) or string_value
string_value = minify == "css" and extra.css_slimmer(string_value) or string_value
string_value = compress == "gzip" and gzip_contents(string_value) or string_value
# iterates over all the target directories for
# to write the contents
for target_directory in target_directories:
# "calculates" the (current) base file path
base_file_directory = os.path.join(directory_path, target_directory)
base_file_directory = os.path.abspath(base_file_directory)
base_file_path = os.path.join(base_file_directory, file_key)
# in case the base file directory does not exists
# it must be created
if not os.path.exists(base_file_directory):
# creates the base file directory
os.makedirs(base_file_directory)
# opens the base file for writing in binary
base_file = open(base_file_path, "wb")
try:
# writes the final string value (after minification
# and compression) to the base file
base_file.write(string_value)
finally:
# closes the base file
base_file.close()
def convert_encoding(
file_path,
source_encoding,
target_encoding,
windows_newline = True,
replacements_list = None
):
"""
Converts the encoding of the specified file.
:type file_path: String
:param file_path: The path to the file to have its encoding converted.
:type source_encoding: String
:param source_encoding: The encoding from which the file is to be converted from.
:type target_encoding: String
:param target_encoding: The encoding to which the file is to be converted.
:type windows_newline: bool
:param windows_newline: If the windows newline should be used.
:type replacements_list: List
:param replacements_list: The list of replacements to perform.
"""
# normalizes the file path and uses it as the path to
# open the reference to it (in reading mode)
file_path_normalized = extra.normalize_path(file_path)
file = open(file_path_normalized, "rb")
try:
# reads the complete string contents from the file and
# checks if the file already has the target encoding
string_value = file.read()
string_value = string_value.replace(b"\r\n", b"\n")
string_value = string_value.replace(b"\r", b"\n")
has_target_encoding = has_encoding(string_value, target_encoding)
# in case the retrieved string value starts with the bom
# (byte order mark) sequence it's removed as it's considered
# deprecated as a method of detecting utf encoding
if string_value.startswith(BOM_SEQUENCE):
string_value = string_value[len(BOM_SEQUENCE):]
# decodes the string value from the specified source encoding, this
# operation may fail as the source encoding may only be a guess on
# the true encoding of the file, the encodes the string value again
# in the target encoding for the file
string_value_decoded = not has_target_encoding and\
string_value.decode(source_encoding) or string_value
string_value_encoded = not has_target_encoding and\
string_value_decoded.encode(target_encoding) or string_value_decoded
# applies the replacements if they're requested to be applied
# so that the final string value is "normalized"
string_value_encoded_replaced = replacements_list and\
apply_replacements_list(string_value_encoded, replacements_list) or\
string_value_encoded
# applies the windows newline if specified, it does so by replacing
# the simple newline character with the windows specific newline
string_value_encoded_replaced = windows_newline and\
string_value_encoded_replaced.replace(b"\n", b"\r\n") or\
string_value_encoded_replaced
finally:
# closes the file for reading (as it's not longer required)
file.close()
# opens the file for writing then writes the file string value
# with the proper string values replaced and re-encoded into the
# target character encoding (as expected)
file = open(file_path_normalized, "wb")
try: file.write(string_value_encoded_replaced)
finally: file.close()
def remove_trailing_newlines(file_path, windows_newline = True):
"""
Removes the extra newlines in the file with the given
file path.
The extra argument defines if the newline format used
should be the windows mode (carriage return and newline).
:type file_path: String
:param file_path: The path to the file to have the trailing
newlines removed.
:type windows_newline: bool
:param windows_newline: If the windows newline should be used.
"""
# normalizes the file path and uses the resulting
# file path as the basis for the opening of the file
file_path_normalized = extra.normalize_path(file_path)
file = open(file_path_normalized, "rb")
try:
# creates a string buffer for buffering
string_buffer = legacy.BytesIO()
# reads the complete set of file lines and then
# reverses their order to detect the end of file
file_lines = file.readlines()
file_lines.reverse()
# start the index
index = 0
# iterates over all the lines in the file
for line in file_lines:
# in case the line is not just a newline character
# breaks the current loop as no extra newlines exist
if not line == b"\n" and not line == b"\r\n": break
# decrements the index
index -= 1
# reverses the file lines
file_lines.reverse()
# in case the original index is set (no extra new lines found)
# all the lines are considered valid, otherwise only the valid
# (non extra lines) are used as valid
if index == 0: valid_file_lines = file_lines
else: valid_file_lines = file_lines[:index]
# iterates over all the file lines
for valid_file_line in valid_file_lines:
# strips the valid file line
valid_file_line_stripped = valid_file_line.rstrip()
# writes the valid file line stripped to the string buffer
string_buffer.write(valid_file_line_stripped)
# in case the newline mode is of type windows, writes the
# carriage return character and the new line character,
# otherwise, writes only the new line character
if windows_newline: string_buffer.write(b"\r\n")
else: string_buffer.write(b"\n")
finally:
# closes the file for reading
file.close()
# retrieves the string value from the string buffer
string_value = string_buffer.getvalue()
# opens the file for writing and outputs the complete
# set of normalized generated contents into it
file = open(file_path_normalized, "wb")
try: file.write(string_value)
finally: file.close()
