def remove_trailing_spaces(file_path, tab_to_spaces, windows_newline = True): """ Removes the extra spaces or tabs in every line of the file contents. The extra argument defines if the newline format used should be the windows mode (carriage return and newline). :type file_path: String :param file_path: The path to the file to have the trailing spaces removed. :type tab_to_spaces: bool :param tab_to_spaces: If the tab characters should be converted to spaces. :type windows_newline: bool :param windows_newline: If the windows newline should be used. """ # normalizes the file path file_path_normalized = extra.normalize_path(file_path) # opens the file for reading file = open(file_path_normalized, "rb") try: # creates a string buffer for buffering string_buffer = legacy.BytesIO() # iterates over all the lines in the file for line in file: # strips the line line_stripped = line.rstrip() # in case the tab must be replaced with spaces # performs the operations with the proper way if tab_to_spaces: line_stripped = line_stripped.replace(b"\t", SPACE_TAB) # writes the stripped line to the string buffer string_buffer.write(line_stripped) # writes the proper line ending sequence taking into # account if the windows newline should be used or not if windows_newline: string_buffer.write(b"\r\n") else: string_buffer.write(b"\n") finally: # closes the file for reading file.close() # retrieves the string value from the string buffer # this should be a bytes based buffer string string_value = string_buffer.getvalue() # opens the file for writing and writes the complete # set of generated contents into it (output operation) file = open(file_path_normalized, "wb") try: file.write(string_value) finally: file.close()
def join_files(file_path): """ Runs the joining operation according to the specification provided by the file located at the provided path. The joining operation may be complex and may take some time until it's completely finished. :type file_path: String :param file_path: The path to the file that contains the json based specification for the joining operation. """ # normalizes the file path value so that it # represents a correct file path file_path_normalized = extra.normalize_path(file_path) # opens the file for reading and then reads the # complete set of data contained in it file = open(file_path_normalized, "rb") try: file_contents = file.read() finally: file.close() # uses the default encoding for json to decode # the complete set of contents in the file file_contents = file_contents.decode("utf-8") # loads the file contents, retrieving the # map of files to be created from joining files_map = json.loads(file_contents) # retrieves the directory path from the normalized # file path (for context resolution) directory_path = os.path.dirname(file_path_normalized) # tries to retrieve the target directories target_directories = files_map.get("_target_directories", None) # in case the target directories are # successfully retrieved if target_directories: # removes the target directories from # the files map (avoids collision) del files_map["_target_directories"] # otherwise the target directories to be # used are the default ones else: # sets the default target directories target_directories = (".",) # iterates over all the files (composition of joined files) # from the files map for file_key, file_value in files_map.items(): # creates the string buffer for temporary # file holding, this is bytes based string_buffer = legacy.BytesIO() # retrieves the current value attributes # (setting default values) files = file_value.get("files", []) minify = file_value.get("minify", None) compress = file_value.get("compress", None) # sets the is first flag is_first = True # iterates over all the files, # to be joined for file in files: # in case it's the first file # no need to write the separator if is_first: is_first = False # otherwise the separator must # be written else: string_buffer.write(b"\r\n") # retrieves the complete file path by joining the # directory path and the current "file" file_path = os.path.join(directory_path, file) file_path = os.path.abspath(file_path) # in case the file does not exists, raises an # error indicating that there was an error if not os.path.exists(file_path): raise RuntimeError("the file path does not exist for file '%s'" % file) # opens the current file for reading # in binary format and reads the complete # set of contents into the current buffer _file = open(file_path, "rb") try: file_contents = _file.read() finally: _file.close() # writes the file contents into the string # buffer, appending the contents to the same file string_buffer.write(file_contents) # retrieves the string value from the buffer, this is # a binary (byte based) string and should be used with # the proper care to avoid unwanted results string_value = string_buffer.getvalue() # minifies and compresses the string value according # to the provided specification, some of this operations # may use complex third-party code string_value = minify == "javascript" and extra.javascript_minify(string_value) or string_value string_value = minify == "css" and extra.css_slimmer(string_value) or string_value string_value = compress == "gzip" and gzip_contents(string_value) or string_value # iterates over all the target directories for # to write the contents for target_directory in target_directories: # "calculates" the (current) base file path base_file_directory = os.path.join(directory_path, target_directory) base_file_directory = os.path.abspath(base_file_directory) base_file_path = os.path.join(base_file_directory, file_key) # in case the base file directory does not exists # it must be created if not os.path.exists(base_file_directory): # creates the base file directory os.makedirs(base_file_directory) # opens the base file for writing in binary base_file = open(base_file_path, "wb") try: # writes the final string value (after minification # and compression) to the base file base_file.write(string_value) finally: # closes the base file base_file.close()
def convert_encoding( file_path, source_encoding, target_encoding, windows_newline = True, replacements_list = None ): """ Converts the encoding of the specified file. :type file_path: String :param file_path: The path to the file to have its encoding converted. :type source_encoding: String :param source_encoding: The encoding from which the file is to be converted from. :type target_encoding: String :param target_encoding: The encoding to which the file is to be converted. :type windows_newline: bool :param windows_newline: If the windows newline should be used. :type replacements_list: List :param replacements_list: The list of replacements to perform. """ # normalizes the file path and uses it as the path to # open the reference to it (in reading mode) file_path_normalized = extra.normalize_path(file_path) file = open(file_path_normalized, "rb") try: # reads the complete string contents from the file and # checks if the file already has the target encoding string_value = file.read() string_value = string_value.replace(b"\r\n", b"\n") string_value = string_value.replace(b"\r", b"\n") has_target_encoding = has_encoding(string_value, target_encoding) # in case the retrieved string value starts with the bom # (byte order mark) sequence it's removed as it's considered # deprecated as a method of detecting utf encoding if string_value.startswith(BOM_SEQUENCE): string_value = string_value[len(BOM_SEQUENCE):] # decodes the string value from the specified source encoding, this # operation may fail as the source encoding may only be a guess on # the true encoding of the file, the encodes the string value again # in the target encoding for the file string_value_decoded = not has_target_encoding and\ string_value.decode(source_encoding) or string_value string_value_encoded = not has_target_encoding and\ string_value_decoded.encode(target_encoding) or string_value_decoded # applies the replacements if they're requested to be applied # so that the final string value is "normalized" string_value_encoded_replaced = replacements_list and\ apply_replacements_list(string_value_encoded, replacements_list) or\ string_value_encoded # applies the windows newline if specified, it does so by replacing # the simple newline character with the windows specific newline string_value_encoded_replaced = windows_newline and\ string_value_encoded_replaced.replace(b"\n", b"\r\n") or\ string_value_encoded_replaced finally: # closes the file for reading (as it's not longer required) file.close() # opens the file for writing then writes the file string value # with the proper string values replaced and re-encoded into the # target character encoding (as expected) file = open(file_path_normalized, "wb") try: file.write(string_value_encoded_replaced) finally: file.close()
def remove_trailing_newlines(file_path, windows_newline = True): """ Removes the extra newlines in the file with the given file path. The extra argument defines if the newline format used should be the windows mode (carriage return and newline). :type file_path: String :param file_path: The path to the file to have the trailing newlines removed. :type windows_newline: bool :param windows_newline: If the windows newline should be used. """ # normalizes the file path and uses the resulting # file path as the basis for the opening of the file file_path_normalized = extra.normalize_path(file_path) file = open(file_path_normalized, "rb") try: # creates a string buffer for buffering string_buffer = legacy.BytesIO() # reads the complete set of file lines and then # reverses their order to detect the end of file file_lines = file.readlines() file_lines.reverse() # start the index index = 0 # iterates over all the lines in the file for line in file_lines: # in case the line is not just a newline character # breaks the current loop as no extra newlines exist if not line == b"\n" and not line == b"\r\n": break # decrements the index index -= 1 # reverses the file lines file_lines.reverse() # in case the original index is set (no extra new lines found) # all the lines are considered valid, otherwise only the valid # (non extra lines) are used as valid if index == 0: valid_file_lines = file_lines else: valid_file_lines = file_lines[:index] # iterates over all the file lines for valid_file_line in valid_file_lines: # strips the valid file line valid_file_line_stripped = valid_file_line.rstrip() # writes the valid file line stripped to the string buffer string_buffer.write(valid_file_line_stripped) # in case the newline mode is of type windows, writes the # carriage return character and the new line character, # otherwise, writes only the new line character if windows_newline: string_buffer.write(b"\r\n") else: string_buffer.write(b"\n") finally: # closes the file for reading file.close() # retrieves the string value from the string buffer string_value = string_buffer.getvalue() # opens the file for writing and outputs the complete # set of normalized generated contents into it file = open(file_path_normalized, "wb") try: file.write(string_value) finally: file.close()