def merge(self): """ Merge all files except the license file and the top-level ``package.mo`` .. warning:: This method is experimental. Do not use it without having a backup of your code. This function merges the `IBPSA` library into other Modelica libraries. In the top-level directory of the destination library, this function creates the file `.copiedFiles.txt` that lists all files that have been copied. In subsequent calls, this function deletes all files listed in `.copiedFiles.txt`, then merges the libraries, and creates a new version of `.copiedFiles.txt`. Therefore, if a model is moved in the `IBPSA` library, it will also be moved in the target library by deleting the old file and copying the new file. This function will merge all Modelica files, Modelica scripts, regression results and images. An exception is the file `IBPSA/package.mo`, as libraries typically have their own top-level package file that contains their release notes and version information. When copying the files, all references and file names that contain the string `IBPSA` will be renamed with the name of the top-level directory of the destination library. Afterwards, the `package.order` files will be regenerated, which allows libraries to have Modelica classes in the same directory as are used by the `IBPSA` library, as long as their name differs. A typical usage is >>> import buildingspy.development.merger as m >>> import os >>> home = os.path.expanduser("~") >>> root = os.path.join(home, "test") >>> ibpsa_dir = os.path.join(root, "modelica", "IBPSA") >>> dest_dir = os.path.join(root, "modelica-buildings", "Buildings") >>> mer = m.IBPSA(ibpsa_dir, dest_dir) # doctest: +SKIP >>> mer.merge() # doctest: +SKIP """ import os import shutil import buildingspy.development.refactor as r # path where a list of all copied files is saved copFilPat = os.path.join(self._target_home, ".copiedFiles.txt") # Build a list of files that were previously merged. # If these file will not be merged again, then we will # delete them. previouslyCopiedFiles = list() if os.path.isfile(copFilPat): roo = self._target_home.rsplit(self._new_library_name, 1)[0] with open(copFilPat, mode="r", encoding="utf-8-sig") as fp: files = fp.read().splitlines() for fil in files: fil = os.path.normpath(fil.rstrip()) if not fil.startswith('#'): absFil = os.path.join(roo, fil) if os.path.isfile(absFil): previouslyCopiedFiles.append(fil) copiedFiles = list() for root, dirs, files in os.walk(self._ibpsa_home, topdown=True): # Exclude certain folders dirs[:] = [d for d in dirs if d not in self._excluded_packages] dirs[:] = [os.path.join(root, d) for d in dirs] # Exclude certain files files = [os.path.join(root, f) for f in files] files = [f for f in list(files) if f not in self._excluded_files] # Location of reference results ref_res = os.path.join(self._target_home, "Resources", "ReferenceResults", "Dymola") for fil in files: srcFil = os.path.join(root, fil) # Loop over all # - .mo files except for top-level .mo file # - .mos files # - ReferenceResults # - OpenModelica/compareVars, as they are autogenerated if os.path.join("OpenModelica", "compareVars") not in srcFil: desFil = srcFil.replace(self._ibpsa_home, self._target_home) desPat = os.path.dirname(desFil) if not os.path.exists(desPat): os.makedirs(desPat) # Process file. # Copy mo and mos files, and replace the library name if desFil.endswith(".mo") or desFil.endswith(".mos"): copiedFiles.append(desFil) self._copy_mo_and_mos(srcFil, desFil) # Only copy reference results if no such file exists. # If a reference file already exists, then don't change it. # This requires to replace # the name of the library in names of the result file elif desFil.startswith(ref_res): dir_name = os.path.dirname(desFil) base_name = os.path.basename(desFil) # Check if the file needs be skipped because it is from an excluded package. skip = False for pac in self._excluded_packages: if "{}_{}".format(self._src_library_name, pac) in base_name: skip = True if not skip: new_file = os.path.join(dir_name, base_name.replace(self._src_library_name, self._new_library_name)) if not os.path.isfile(new_file): copiedFiles.append(new_file) shutil.copy2(srcFil, new_file) # Copy all other files. This may be images, C-source, libraries etc. else: copiedFiles.append(desFil) shutil.copy2(srcFil, desFil) # Delete the files that were previously merged, but are no longer in IBPSA. # First, remove from the list the files that were copied just now for fil in copiedFiles: filNam = self._new_library_name + fil.split(self._target_home)[1] if previouslyCopiedFiles.count(filNam) > 0: previouslyCopiedFiles.remove(filNam) # Now, remove the files, unless they are no longer in the repository anyway. for fil in previouslyCopiedFiles: filNam = os.path.join( self._target_home[0: self._target_home.rfind(self._new_library_name)], fil) if os.path.isfile(filNam): os.remove(filNam) # Generate package.order files r.write_package_order(self._target_home, True) # Save a list of all files that were copied. with open(copFilPat, mode="w", encoding="utf-8") as fp: fp.write("# Do not edit this file unless you know what you are doing.\n") fp.write("# This file is used by the IBPSA merge script and generated by BuildingsPy.\n") for fil in sorted(copiedFiles): fp.write(self._new_library_name + fil.split(self._target_home)[1] + "\n")
def merge(self, overwrite_reference_results=False): """ Merge all files except the license file and the top-level ``package.mo`` .. warning:: This method is experimental. Do not use it without having a backup of your code. This function merges the `IBPSA` library into other Modelica libraries. In the top-level directory of the destination library, this function creates the file `.copiedFiles.txt` that lists all files that have been copied. In subsequent calls, this function deletes all files listed in `.copiedFiles.txt`, then merges the libraries, and creates a new version of `.copiedFiles.txt`. Therefore, if a model is moved in the `IBPSA` library, it will also be moved in the target library by deleting the old file and copying the new file. This function will merge all Modelica files, Modelica scripts, regression results and images. An exception is the file `IBPSA/package.mo`, as libraries typically have their own top-level package file that contains their release notes and version information. When copying the files, all references and file names that contain the string `IBPSA` will be renamed with the name of the top-level directory of the destination library. Afterwards, the `package.order` files will be regenerated, which allows libraries to have Modelica classes in the same directory as are used by the `IBPSA` library, as long as their name differs. :param overwrite_reference_results: Boolean `True` causes the reference results to be overwritten, `False` causes the reference results to be skipped if they already exist. Note that if the reference result does not yet exist, it will be merged regardless of the setting of this parameter. A typical usage is >>> import buildingspy.development.merger as m >>> import os >>> home = os.path.expanduser("~") >>> root = os.path.join(home, "test") >>> ibpsa_dir = os.path.join(root, "modelica", "IBPSA") >>> dest_dir = os.path.join(root, "modelica-buildings", "Buildings") >>> mer = m.IBPSA(ibpsa_dir, dest_dir) # doctest: +SKIP >>> mer.merge() # doctest: +SKIP """ import os import shutil import fnmatch import buildingspy.development.refactor as r # path where a list of all copied files is saved copFilPat = os.path.join(self._target_home, ".copiedFiles.txt") # Build a list of files that were previously merged. # If these file will not be merged again, then we will # delete them. previouslyCopiedFiles = list() if os.path.isfile(copFilPat): roo = self._target_home.rsplit(self._new_library_name, 1)[0] with open(copFilPat, mode="r", encoding="utf-8-sig") as fp: files = fp.read().splitlines() for fil in files: fil = os.path.normpath(fil.rstrip()) if not fil.startswith('#'): absFil = os.path.join(roo, fil) if os.path.isfile(absFil): previouslyCopiedFiles.append(fil) # Build list of files to copy, and store them in filesToCopy filesToCopy = list() for root, dirs, files in os.walk(self._ibpsa_home, topdown=True): dirs[:] = [d for d in dirs if d not in self._excluded_packages] for file in files: filesToCopy.append(os.path.join(root, file)) for pattern in self._excluded_files: exclude_files = IBPSA.filter_files(filesToCopy, pattern) for f in exclude_files: filesToCopy.remove(f) # Location of reference results ref_res = os.path.join(self._target_home, "Resources", "ReferenceResults", "Dymola") # Location of heat pump calibration hea_pum = os.path.join(self._target_home, "Resources", "src", "fluid", "heatpumps", "calibration") # Iterate over the list of files to be copied. copiedFiles = list() for fil in filesToCopy: srcFil = os.path.join(root, fil) # Loop over all # - .mo files except for top-level .mo file # - .mos files # - ReferenceResults # - OpenModelica/compareVars, as they are autogenerated if os.path.join("OpenModelica", "compareVars") not in srcFil: desFil = srcFil.replace(self._ibpsa_home, self._target_home) desPat = os.path.dirname(desFil) if not os.path.exists(desPat): os.makedirs(desPat) # Process file. # Copy mo and mos files, and replace the library name if desFil.endswith(".mo") or desFil.endswith(".mos"): copiedFiles.append(desFil) self._copy_mo_and_mos(srcFil, desFil) # Only copy reference results if no such file exists. # If a reference file already exists, then don't change it. # This requires to replace # the name of the library in names of the result file elif desFil.startswith(ref_res): dir_name = os.path.dirname(desFil) base_name = os.path.basename(desFil) # Check if the file needs be skipped because it is from an excluded package. skip = False for pac in self._excluded_packages: if "{}_{}".format(self._src_library_name, pac) in base_name: skip = True if not skip: new_file = os.path.join( dir_name, base_name.replace(self._src_library_name, self._new_library_name)) # if overwrite_reference_results, the copy of reference files # is forced if overwrite_reference_results: copiedFiles.append(new_file) shutil.copy2(srcFil, new_file) else: if not os.path.isfile(new_file): copiedFiles.append(new_file) shutil.copy2(srcFil, new_file) # elif desFil.startswith(hea_pum): dir_name = os.path.dirname(desFil) base_name = os.path.basename(desFil) new_file = os.path.join( dir_name, base_name.replace(self._src_library_name, self._new_library_name)) if not os.path.isfile(new_file): copiedFiles.append(new_file) rep = dict() rep[self._src_library_name] = self._new_library_name # If it is a binary file, simply copy it. Otherwise, the copy/replace/write # will read, search for a string and rewrite the file, leading to a Unicode error. # This happens for example for the file # build/html/_static/bootstrap-3.3.6/fonts/glyphicons-halflings-regular.woff2 bin_ext = [".woff", ".woff2", ".ttf", ".eot"] is_binary = False for ext in bin_ext: if srcFil.endswith(ext): is_binary = True break if is_binary: copyfile(srcFil, new_file) else: self._copy_rename(srcFil, new_file, rep) # Copy all other files. This may be images, C-source, libraries etc. else: copiedFiles.append(desFil) shutil.copy2(srcFil, desFil) # Delete the files that were previously merged, but are no longer in IBPSA. # First, remove from the list the files that were copied just now for fil in copiedFiles: filNam = self._new_library_name + fil.split(self._target_home)[1] if previouslyCopiedFiles.count(filNam) > 0: previouslyCopiedFiles.remove(filNam) # Now, remove the files, unless they are no longer in the repository anyway. for fil in previouslyCopiedFiles: filNam = os.path.join( self._target_home[0:self._target_home. rfind(self._new_library_name)], fil) if os.path.isfile(filNam): os.remove(filNam) # Generate package.order files r.write_package_order(self._target_home, True) # Save a list of all files that were copied. with open(copFilPat, mode="w", encoding="utf-8") as fp: fp.write( "# Do not edit this file unless you know what you are doing.\n" ) fp.write( "# This file is used by the IBPSA merge script and generated by BuildingsPy.\n" ) for fil in sorted(copiedFiles): fp.write(self._new_library_name + fil.split(self._target_home)[1] + "\n")
def merge(self): """ Merge all files except the license file and the top-level ``package.mo`` .. warning:: This method is experimental. Do not use it without having a backup of your code. This function merges the `Annex60` library into other Modelica libraries. In the top-level directory of the destination library, this function creates the file `.copiedFiles.txt` that lists all files that have been copied. In subsequent calls, this function deletes all files listed in `.copiedFiles.txt`, then merges the libraries, and creates a new version of `.copiedFiles.txt`. Therefore, if a model is moved in the `Annex60` library, it will also be moved in the target library by deleting the old file and copying the new file. This function will merge all Modelica files, Modelica scripts, regression results and images. An exception is the file `Annex60/package.mo`, as libraries typically have their own top-level package file that contains their release notes and version information. When copying the files, all references and file names that contain the string `Annex60` will be renamed with the name of the top-level directory of the destination library. Afterwards, the `package.order` files will be regenerated, which allows libraries to have Modelica classes in the same directory as are used by the `Annex60` library, as long as their name differs. A typical usage is >>> import buildingspy.development.merger as m >>> import os >>> home = os.path.expanduser("~") >>> root = os.path.join(home, "test") >>> annex60_dir = os.path.join(root, "modelica-annex60", "Annex60") >>> dest_dir = os.path.join(root, "modelica-buildings", "Buildings") >>> mer = m.Annex60(annex60_dir, dest_dir) # doctest: +SKIP >>> mer.merge() # doctest: +SKIP """ import os import shutil import re import buildingspy.development.refactor as r # path where a list of all copied files is saved copFilPat = os.path.join(self._target_home, ".copiedFiles.txt") # Build a list of files that were previously merged. # If these file will not be merged again, then we will # delete them. previouslyCopiedFiles = list() if os.path.isfile(copFilPat): roo = self._target_home.rsplit(self._new_library_name, 1)[0] with open(copFilPat, 'r') as fp: files = fp.read().splitlines() for fil in files: fil = os.path.normpath(fil.rstrip()) if not fil.startswith('#'): absFil = os.path.join(roo, fil) if os.path.isfile(absFil): previouslyCopiedFiles.append(fil) copiedFiles = list() # Location of reference results ref_res = os.path.join(self._target_home, "Resources", "ReferenceResults", "Dymola") excluded_dirs = '|'.join([x for x in (self._excluded_packages)]) or r'$.' excluded_files = '|'.join( ["{}_{}_".format("Annex60", x) for x in (self._excluded_packages)]) or r'$.' for root, dirs, files in os.walk(self._annex60_home): # Exclude certain folders dirs[:] = [os.path.join(root, d) for d in dirs] dirs[:] = [d for d in dirs if not re.search(excluded_dirs, d)] # Exclude certain files files = [os.path.join(root, f) for f in files] files = [f for f in files if not re.search(excluded_files, f)] for fil in files: srcFil = os.path.join(root, fil) # Loop over all # - .mo files except for top-level .mo file # - .mos files # - ReferenceResults # - OpenModelica/compareVars, as they are autogenerated if (not srcFil.endswith(os.path.join("Annex60", "package.mo")) \ or srcFil.endswith("legal.html"))\ and os.path.join("OpenModelica", "compareVars") not in srcFil: desFil = srcFil.replace(self._annex60_home, self._target_home) desPat = os.path.dirname(desFil) if not os.path.exists(desPat): os.makedirs(desPat) # Process file. # Copy mo and mos files, and replace the library name if desFil.endswith(".mo") or desFil.endswith(".mos"): copiedFiles.append(desFil) self._copy_mo_and_mos(srcFil, desFil) # Only copy reference results if no such file exists. # If a reference file already exists, then don't change it. # This requires to replace # the name of the library in names of the result file elif desFil.startswith(ref_res): dir_name = os.path.dirname(desFil) base_name = os.path.basename(desFil) new_file = os.path.join( dir_name, base_name.replace("Annex60", self._new_library_name)) if not os.path.isfile(new_file): copiedFiles.append(new_file) shutil.copy2(srcFil, new_file) # Copy all other files. This may be images, C-source, libraries etc. else: copiedFiles.append(desFil) shutil.copy2(srcFil, desFil) # Delete the files that were previously merged, but are no longer in Annex60. # First, remove from the list the files that were copied just now for fil in copiedFiles: filNam = self._new_library_name + fil.split(self._target_home)[1] if previouslyCopiedFiles.count(filNam) > 0: previouslyCopiedFiles.remove(filNam) # Now, remove the files, unless they are no longer in the repository anyway. for fil in previouslyCopiedFiles: filNam = os.path.join( self._target_home[0:self._target_home. rfind(self._new_library_name)], fil) if os.path.isfile(filNam): os.remove(filNam) # Generate package.order files r.write_package_order(self._target_home, True) # Save a list of all files that were copied. with open(copFilPat, 'w') as fp: fp.write( "# Do not edit this file unless you know what you are doing.\n" ) fp.write( "# This file is used by the Annex60 merge script and generated by BuildingsPy.\n" ) for fil in sorted(copiedFiles): fp.write(self._new_library_name + fil.split(self._target_home)[1] + "\n")