def find_cmake_tags(base_path, release, project_path): ## @brief Find the tags that went into a CMake release found # at the path specified # @param base_path Starting base path for the release number and flavour # @param release The Athena release number # @param project_path The path element inside each project where the # project is installed release_packages = {} project_directories = [ dir for dir in os.listdir(base_path) if dir.startswith("Atlas") or dir == "DetCommon" ] for project in project_directories: packages_file = os.path.join(base_path, project, release, project_path, "packages.txt") if not os.path.exists(packages_file): logger.warning( "Project packages file {0} doesn't exist - skipping this project" .format(packages_file)) continue project_packages = read_project_packages(packages_file, project) logger.debug("Found {0} packages for project {1}".format( len(project_packages), project)) release_packages.update(project_packages) logger.info("Found {0} packages in release {1}".format( len(release_packages), release)) return release_packages
def get_tags(tag_files, svn_path_accept): ## @brief Parse packages and package tags from release diff files # @param tag_files List of release tag files to query # @param svn_path_accept List of paths to filter on # @return dictionary keyed by package (including path) and value as sorted list of tags svn_package_tags = {} for tag_file in tag_files: with open(tag_file) as tag_fh: tag_dict = json.load(tag_fh) logger.info("Getting tag lists from {0}".format( tag_dict["release"]["name"])) for package, package_info in tag_dict["tags"].iteritems(): if len(svn_path_accept) > 0: accept = False for path in svn_path_accept: if package.startswith(path): accept = True break if not accept: continue svn_tag = package_info["svn_tag"] if svn_tag != "trunk": svn_tag = os.path.join("tags", svn_tag) if package in svn_package_tags: svn_package_tags[package].add(svn_tag) else: svn_package_tags[package] = set((svn_tag, )) # Now convert back to list and sort tags... for package in svn_package_tags: svn_package_tags[package] = list(svn_package_tags[package]) svn_package_tags[package].sort() return svn_package_tags
def find_nicos_from_base(nicos_path, base_release): ## @brief Find base release and cache release tag files when only a base release number # is given # @param nicos_path Base path to NICOS tag file area # @param base_release Base release number A.B or A.B.X (e.g., 21.0[.1]) # @return list of matching tag files, in release numbered order release_list = [] dir_list = os.listdir(nicos_path) release_match = "{0}(\.(\d+))*$".format( os.path.basename(base_release).replace(".", r"\.")) logger.debug( "Matching releases against pattern '{0}'".format(release_match)) for entry in dir_list: if re.match(release_match, entry): release_list.append(entry) logger.debug("Matching releases: {0}".format(release_list)) # It's not actually necessary to sort the releases, but it does no harm release_list.sort(cmp=release_compare) logger.info( "Found ordered list of production caches: {0}".format(release_list)) tag_files = [] for release in release_list: tag_files.append(get_tag_file(os.path.join(nicos_path, release))) return tag_files
def remove_gpl_block(source_lines, keep_lines, depth): '''Get rid of GPL license block''' gpl_idx = search_from(source_lines, "GNU General Public License", 0, depth) if gpl_idx == -1: return # OK, found a matching line, now search for start and end of block start_idx = search_from(source_lines, r"\*\*\*\*\*\*\*\*", gpl_idx, max=10, backwards=True) stop_idx = search_from(source_lines, r"\*\*\*\*\*\*\*\*", gpl_idx, max=10, backwards=False) if start_idx == -1 or stop_idx == -1: logger.warning( "Found GPL trigger line, but failed to find start/end of license block" ) return logger.info( "Found GPL license block to suppress from lines {0} to {1}".format( start_idx + 1, stop_idx + 1)) for idx in range(start_idx, stop_idx + 1): keep_lines[idx] = False
def load_exceptions_file(filename, reject_changelog=False): ## @brief Parse and return path globbing exceptions file # @param filename File containing exceptions # @param reject_changelog Special flag used by svnpull to ensure that # ChangeLog files are rejected (in a normal svn2git they are accepted, # onto the import branches, but then excluded specially from the # release branches) # @return Tuple of path globs to accept and globs to reject, converted to regexps path_accept = [] path_reject = [] if filename != "NONE": with open(filename) as filter_file: logger.info("Loaded import exceptions from {0}".format(filename)) for line in filter_file: line = line.strip() if reject_changelog and ("ChangeLog" in line): logger.debug( "Found ChangeLog line, which will be forced to reject: {0}" .format(line)) line = "- */ChangeLog" if line.startswith("#") or line == "": continue if line.startswith("-"): path_reject.append( re.compile(fnmatch.translate(line.lstrip("- ")))) else: path_accept.append( re.compile(fnmatch.translate(line.lstrip("+ ")))) logger.debug("Glob accept: {0}".format([m.pattern for m in path_accept])) logger.debug("Glob reject: {0}".format([m.pattern for m in path_reject])) return path_accept, path_reject
def initialise_metadata(cachefile): ## @brief Load existing cache file, if it exists, or return empty cache # @param cachefile Name of cache file (serialised in JSON) if os.path.exists(cachefile): logger.info("Reloading cache from {0}".format(cachefile)) with file(cachefile) as md_load: svn_metadata_cache = json.load(md_load) else: svn_metadata_cache = {} return svn_metadata_cache
def init_git(gitrepo): ## @brief Initialise git repo, if needed # @param gitrepo Git repository path if not os.path.exists(gitrepo): os.makedirs(gitrepo) os.chdir(gitrepo) if os.path.exists(os.path.join(gitrepo, ".git")): logger.info("Found existing git repo, {0}".format(gitrepo)) check_output_with_retry(("git", "reset", "--hard")) else: logger.info("Initialising git repo: {0}".format(gitrepo)) check_output_with_retry(("git", "init"))
def find_youngest_tag(tag_diff, svn_metadata_cache): '''Use the svn metadata cache to find the youngest tag in the release''' yougest_tag = None youngest_svn_revision = -1 if svn_metadata_cache: for package, tag in tag_diff[0]["diff"]["add"].iteritems(): if (package in svn_metadata_cache and svn_metadata_cache[package][os.path.join("tags", tag)]["revision"] > youngest_svn_revision): youngest_svn_revision = svn_metadata_cache[package][os.path.join("tags", tag)]["revision"] yougest_tag = os.path.join("import", "tag", tag) logger.info("Tag to branch from master at is {0} (SVN revision {1})".format(yougest_tag, youngest_svn_revision)) return yougest_tag
def remove_plain_copyright(source_lines, keep_lines, depth): '''Get rid of plain vanilla copyright lines''' idx = 0 copyre = re.compile(r"//.*copyright.*\([Cc]\)") for line in source_lines: if copyre.search(line): logger.info( "Found copyright line to suppress at index {0}: {1}".format( idx + 1, line)) keep_lines[idx] = False if idx > depth: break idx += 1
def prepare_branch_point(branch, parentbranch=None): ## @brief Using information about the target branch and any parent # switch to the correct point in history to start/continue # @param branch Target branch name # @param parentbranch If creating a new branch, this is the @c BRANCH:COMMIT_ID of # where to make the new branch from; syntax @c BRANCH:@FILE and @c BRANCH:@TIMESTAMP # is also supported, where the timestamp will be used to find the branch directly # (and can be taken from @c JSON release data in @c FILE) if not parentbranch or branch_exists(branch): logger.info("Switching to branch {0}".format(branch)) switch_to_branch(branch, orphan=True) else: parent, commit = parentbranch.split(":") check_output_with_retry(("git", "checkout", parent), retries=1) # needed? if commit.startswith("@"): timestamp = commit[1:] # If this maps to a file, try to open it as a release JSON, otherwise treat it as # a plain timestamp if os.access(timestamp, os.F_OK): logger.info("Taking branching timestamp from file {0}".format(timestamp)) with open(timestamp) as fh: branch_point_release = json.load(fh) timestamp = branch_point_release["release"]["timestamp"] logger.info("Using timestamp {0} for branch point".format(timestamp)) commit = check_output_with_retry(["git", "log", "--until", str(timestamp), "-n1", "--pretty=format:%H"], retries=1).strip() logger.info("Mapped timestamp {0} to commit {1}".format(timestamp, commit)) check_output_with_retry(("git", "checkout", commit), retries=1) check_output_with_retry(("git", "checkout", "-b", branch), retries=1)
def main(): parser = argparse.ArgumentParser( description= 'ATLAS CMT tag parser, grabbing tag content for a CMT cache release. ' 'This is quite a hacky script, only filling in the gaps in NICOS knowledge for ' 'ATLAS P1HLT caches.') parser.add_argument('release', metavar='RELEASE', help="CMT requirements file to parse") parser.add_argument( '--tagdir', default="tagdir", help= "output directory for tag files, each release will generate an entry here (default \"tagdir\")" ) parser.add_argument( '--overwrite', action="store_true", default=False, help= "Overwrite any exisitng configuration files (otherwise, just skip over)" ) parser.add_argument('--debug', '--verbose', "-v", action="store_true", help="switch logging into DEBUG mode") args = parser.parse_args() if args.debug: logger.setLevel(logging.DEBUG) release_description = parse_release_data(args.release) release_tags = parse_tag_file(args.release) logger.info("Processing tags for release {0}".format( release_description["name"])) output_file = os.path.join(args.tagdir, release_description["name"]) if args.overwrite or not os.path.exists(output_file): with open(os.path.join(args.tagdir, release_description["name"]), "w") as tag_output: json.dump({ "release": release_description, "tags": release_tags }, tag_output, indent=2) else: logger.debug( "Skipped writing to {0} - overwrite is false".format(output_file))
def svn_get_path_metadata(svnroot, package, package_path, revision=None): # # @brief Get SVN metadata and return as a simple dictionary keyed on date, author and commit revision logger.info("Querying SVN metadata for {0}".format(os.path.join(package, package_path))) cmd = ["svn", "info", os.path.join(svnroot, package, package_path), "--xml"] svn_info = check_output_with_retry(cmd) tree = eltree.fromstring(svn_info) info = {"date": tree.find(".//date").text.rsplit(".",1)[0], # Strip off sub-second part "author": tree.find(".//author").text, "revision": tree.find(".//commit").attrib['revision']} cmd = ["svn", "log", os.path.join(svnroot, package, package_path), "-r", info["revision"], "--xml"] svn_log = check_output_with_retry(cmd) tree = eltree.fromstring(svn_log) info["msg"] = tree.find(".//msg").text.strip().encode('ascii', 'ignore') return info
def svn_cleanup(svn_path, svn_co_root, svn_path_accept=[], svn_path_reject=[]): # # @brief Cleanout files we do not want to import into git # @param svn_path Full path to checkout of SVN package # @param svn_co_root Base directory of SVN checkout # @param svn_path_accept List of file path globs to always import to git # @param svn_path_reject List of file path globs to never import to git # File size veto for root, dirs, files in os.walk(svn_path): if ".svn" in dirs: shutil.rmtree(os.path.join(root, ".svn")) dirs.remove(".svn") for name in files: filename = os.path.join(root, name) svn_filename = filename[len(svn_co_root) + 1:] path_accept_match = False for filter in svn_path_accept: if re.match(filter, svn_filename): logger.info("{0} imported from globbed exception {1}".format(svn_filename, filter.pattern)) path_accept_match = True break if path_accept_match: continue try: # Rejection always takes precedence for filter in svn_path_reject: if re.match(filter, svn_filename): logger.info("{0} not imported due to {1} filter".format(svn_filename, filter.pattern)) os.remove(filename) continue if os.lstat(filename).st_size > 100 * 1024: if "." in name and name.rsplit(".", 1)[1] in ("cxx", "py", "h", "java", "cc", "c", "icc", "cpp", "hpp", "hh", "f", "F"): logger.info("Source file {0} is too large, but importing anyway (source files always imported)".format(filename)) else: logger.info("File {0} is too large - not importing".format(filename)) os.remove(filename) continue if name.startswith("."): logger.info("File {0} starts with a '.' - not importing".format(filename)) os.remove(filename) continue except OSError, e: logger.debug("Got OSError (usually harmless) treating {0}: {1}".format(filename, e))
def main(): parser = argparse.ArgumentParser( description='Merge releases to create a super-release') parser.add_argument('targetrelease', metavar='RELEASE', help="Target release") parser.add_argument('mergerelease', metavar='RELEASE', nargs="+", help="Releases to merge into target") parser.add_argument('--debug', '--verbose', "-v", action="store_true", help="switch logging into DEBUG mode") args = parser.parse_args() if args.debug: logger.setLevel(logging.DEBUG) with open(args.targetrelease) as target: target_release_data = json.load(target) for release in args.mergerelease: with open(release) as merge: merge_release_data = json.load(merge) for package_path, package_data in merge_release_data["tags"].iteritems( ): if package_path not in target_release_data["tags"]: target_release_data["tags"][package_path] = package_data logger.info("Merged {0} at tag {1} from {2}".format( package_path, package_data["svn_tag"], release)) else: logger.debug("Package {0} already exists in target".format( package_path)) try: os.rename(args.targetrelease, args.targetrelease + ".bak") with open(args.targetrelease, "w") as output_fh: json.dump(target_release_data, output_fh, indent=2) except OSError, e: logger.error("Error while rewriting target file {0}: {1}".format( args.targetrelease, e))
def relicense(fname, license=True, rename=False, depth=20): logger.info("Processing file {0}".format(fname)) tmpname = fname + ".relicense" fmode = os.stat(fname).st_mode with open(fname) as infile, open(tmpname, "w") as outfile: source_lines = infile.readlines() keep_lines = [True for l in range(len(source_lines))] remove_plain_copyright(source_lines, keep_lines, depth) remove_gpl_block(source_lines, keep_lines, depth) if license: add_standard_license(fname, source_lines, keep_lines) write_relicensed_file(source_lines, keep_lines, outfile) os.chmod(tmpname, fmode) if rename: os.rename(fname, fname + ".bak") os.rename(tmpname, fname)
def backskip_filter(tagfiles): ## @brief Reverse order parse a set of tagfiles and reject those # where procesing would require a backskip in time # @param tagfiles List of tag files # #return Filtered list of tagfiles with backskips removed last_timestamp = 0 last_release = "" tagfiles_copy = tagfiles[:] tagfiles_copy.reverse() for tagfile in tagfiles_copy: with open(tagfile) as tagfile_fh: release_data = json.load(tagfile_fh) if last_timestamp and release_data["release"]["timestamp"] > last_timestamp: logger.info("Vetoing release {0} (from {1}) because of " "backskip for {2}".format(release_data["release"]["name"], tagfile, last_release)) tagfiles.remove(tagfile) continue last_timestamp = release_data["release"]["timestamp"] last_release = release_data["release"]["name"] logger.info("Accepted release {0} at {1}".format(last_release, last_timestamp)) return tagfiles
def scan_svn_tags_and_get_metadata(svnroot, svn_packages, svn_metadata_cache, author_metadata_cache, all_package_tags=False, package_veto=[]): # # @brief Get SVN metadata for each of the package tags we're interested in # @param svnroot URL of SVN repository # @param svn_packages Dictionary of packages and tags to process # @param svn_metadata_cache SVN metadata cache # @param author_metadata_cache author metadata cache with name and email for commits # @param all_package_tags Boolean flag triggering import of all package tags in SVN # @param package_veto List of packages to just plain refuse to handle # First we establish the list of tags which we need to deal with. for package, package_tags in svn_packages.iteritems(): logger.info("Preparing package {0} (base tags: {1})".format(package, package_tags)) if all_package_tags: oldest_tag = svn_packages[package][0] tags = get_all_package_tags(svnroot, package) try: package_tags.extend(tags[tags.index(oldest_tag) + 1:]) except ValueError: logger.error("Oldest release tag ({0}) for package {1} not found in SVN!".format(oldest_tag, package)) sys.exit(1) # We need to now sort the package tags and remove any duplicates ordered_tags = list(set(package_tags)) ordered_tags.sort(cmp=svn_tag_cmp) svn_packages[package] = ordered_tags # Now iterate over the required tags and ensure we have the necessary metadata for package, package_tags in svn_packages.iteritems(): if package in package_veto: logger.info("Package {0} is vetoed - skipping SVN metadata import".format(package)) continue package_name = os.path.basename(package) package_path = os.path.dirname(package) for tag in package_tags: # Do we have metadata? if package_name not in svn_metadata_cache: svn_metadata_cache[package_name] = {"path": package_path, "svn": {}} try: if tag == "trunk": # We always need to get the metadata for trunk tags as we need to # know the current revision svn_metadata = svn_get_path_metadata(svnroot, package, tag) if tag not in svn_metadata_cache[package_name]["svn"]: svn_metadata_cache[package_name]["svn"][tag] = {svn_metadata["revision"]: svn_metadata} elif svn_metadata["revision"] not in svn_metadata_cache[package_name]["svn"][tag]: svn_metadata_cache[package_name]["svn"][tag][svn_metadata["revision"]] = svn_metadata elif tag not in svn_metadata_cache[package_name]["svn"]: svn_metadata = svn_get_path_metadata(svnroot, package, tag) svn_metadata_cache[package_name]["svn"][tag] = {svn_metadata["revision"]: svn_metadata} else: svn_metadata = svn_metadata_cache[package_name]["svn"][tag].values()[0] if svn_metadata["author"] not in author_metadata_cache: try: author_metadata_cache[svn_metadata["author"]] = author_info_lookup(svn_metadata["author"]) except RuntimeError, e: logger.info("Failed to get author information for {0}: {1}".format(package, e)) author_metadata_cache[svn_metadata["author"]] = {"name": svn_metadata["author"], "email": "{0}@cern.ch".format(svn_metadata["author"])} except RuntimeError: logger.warning("Failed to get SVN metadata for {0}".format(os.path.join(package, tag)))
def check_output_with_retry(cmd, retries=2, wait=10, ignore_fail=False, dryrun=False): ## @brief Multiple attempt wrapper for subprocess.check_call (especially remote SVN commands can bork) # @param cmd list or tuple of command line parameters # @param retries Number of attempts to execute successfully # @param wait Sleep time after an unsuccessful execution attempt # @param ignore_fail Do not raise an exception if the command fails # @param dryrun If @c True do not actually execute the command, only print it and return an empty string # @return String containing command output if dryrun: logger.info("Dryrun mode: {0}".format(cmd)) return "" success = failure = False tries = 0 start = time.time() while not success and not failure: tries += 1 try: logger.debug("Calling {0}".format(cmd)) output = subprocess.check_output(cmd) success = True except subprocess.CalledProcessError: if ignore_fail: success = True output = "" continue logger.warning("Attempt {0} to execute {1} failed".format( tries, cmd)) if tries > retries: failure = True else: time.sleep(wait) if failure: raise RuntimeError("Repeated failures to execute {0}".format(cmd)) logger.debug("Executed in {0}s".format(time.time() - start)) return output
def svn_license_injector(svn_path, svn_co_root, license_text, license_path_accept=[], license_path_reject=[]): ## @brief Add license statements to code before import # @param svn_path Filesystem path to cleaned up SVN checkout # @param svn_co_root Base directory of SVN checkout # @param license_text List of strings that comprise the license to apply # @param license_path_accept Paths to force include in license file addition (NOT IMPLEMENTED YET) # @param license_path_reject Paths to exclude from license file addition # license file addition for root, dirs, files in os.walk(svn_path): for name in files: filename = os.path.join(root, name) svn_filename = filename[len(svn_co_root) + 1:] path_veto = False for filter in license_path_reject: if re.match(filter, svn_filename): logger.info("File {0} will not have a license file applied".format(svn_filename, filter.pattern)) path_veto = True break for filter in license_path_accept: if re.match(filter, svn_filename): logger.info("File {0} will have a license file applied".format(svn_filename, filter.pattern)) path_veto = False break if path_veto: continue # Get the file's mode here to then restore it try: fmode = os.stat(filename).st_mode extension = svn_filename.rsplit(".", 1)[1] if "." in svn_filename else "" if extension in ("cxx", "cpp", "icc", "cc", "c", "C", "h", "hpp", "hh"): inject_c_license(filename, license_text) os.chmod(filename, fmode) elif extension in ("py", "cmake"): inject_py_license(filename, license_text) os.chmod(filename, fmode) except OSError, e: # Can happen if a file is a softlink to nowhere logger.warning("Got an exception on stating {0}: {1}".format(filename, e))
def add_standard_license(fname, source_lines, keep_lines): '''Add the standard CERN license if it is missing''' if search_from( source_lines, r"Copyright \(C\) 2002-2017 CERN for the benefit of the ATLAS collaboration" ) != -1: logger.info("Standard license already present") return extension = fname.rsplit(".", 1)[1] if "." in fname else "" if extension in ("cxx", "cpp", "icc", "cc", "c", "C", "h", "hpp", "hh"): logger.info("Adding C style license") add_c_license(source_lines, keep_lines) elif extension in ("py", "cmake"): logger.info("Adding py style license") add_py_license(source_lines, keep_lines)
def find_packages_for_update(release_data, tag_list, branch, svn_metadata_cache, release_tag_unprocessed, only_forward=False): ## @brief Find packages that need updates, comparing release tag content with # git import tags already made # @param release_data Release tag content dictionary # @param tag_list List of current git tags # @param branch Git branch being constructed # @param svn_metadata_cache Usual cache data for SVN stuff # @param release_tag_unprocessed Dictionary with current "tag" metadata, useful to mark up import # @param only_forward If @c True then never revert a package to a previous version or import a branch tag # @return Tuple of tag import dictionary, and a list of all "processed" packages ## Loop over all packages in a release and see if the package # - is missing from the svn metadata cache (so skip) # - is already imported at current HEAD (so skip) # - is new or updated (so mark for import) # Final construct is the import_list dictionary import_list = {} packages_considered = [] for package, package_data in release_data["tags"].iteritems(): package_name = os.path.basename(package) packages_considered.append(package_name) package_tag = package_data["svn_tag"] if package_name not in svn_metadata_cache: logger.debug("Package {0} not found - assuming restricted import".format(package_name)) continue if package_tag == "trunk": svn_meta_tag_key = "trunk" else: svn_meta_tag_key = os.path.join("tags", package_tag) for revision in svn_metadata_cache[package_name]["svn"][svn_meta_tag_key]: git_import_tag = get_flattened_git_tag(package, package_tag, revision) if git_import_tag not in tag_list: logger.debug("Import tag {0} not found - assuming restricted import".format(git_import_tag)) continue branch_import_tag = get_flattened_git_tag(package, package_tag, revision, branch) logger.debug("Considering import of {0} ({1}@r{2}) to {3} " "for release {4}".format(branch_import_tag, package_tag, revision, branch, release_data["release"]["name"])) if branch_import_tag in tag_list: logger.info("Import of {0} ({1} r{2}) onto {3} done - skipping".format(package, package_tag, revision, branch)) continue if only_forward: if package_name in release_tag_unprocessed and package_compare(release_tag_unprocessed[package_name]["svn_tag"], package_tag) != -1: logger.info("Import of {0} onto {1} is blocked - onlyforward option will not downgrade tags".format(package_tag, branch)) continue ## @note The structure of the dictionary used to direct the import of a package is: # "package": full package path # "package_name": package basename (for convenience) # "git_import_tag": the git import tag that identifies the import of the version of this package # "svn_tag": the SVN tag corresponding to this package version # "svn_revision": the SVN revision for this package version # "branch_import_tag": the git import tag that will be created to stamp this import as done # "svn_meta_tag_key": the key used in the SVN metadata dictionary to get SVN metadata # "current_branch_import_tag": the git tag indicating the current version of this package import_element = {"package": package, "package_name": os.path.basename(package), "git_import_tag": git_import_tag, "svn_tag": package_tag, "svn_revision": revision, "branch_import_tag": branch_import_tag, "svn_meta_tag_key": svn_meta_tag_key, "current_branch_import_tag": release_tag_unprocessed[package_name]["git_tag"] if package_name in release_tag_unprocessed else None} logger.debug("Will import {0} to {1}".format(import_element, branch)) if revision in import_list: import_list[revision].append(import_element) else: import_list[revision] = [import_element] return import_list, packages_considered
def find_cmake_releases(install_path, release, nightly=None, arch=None): ## @brief Find the base path and project sub-path for a CMake release # @param install_path Base release area for CMake installed releases # @param release Athena release series + release flavour number (e.g., 21.0) # @param nightly Nightly series to search (otherwise look for installed release) # @param arch Manually specify architecture # @return Tuple with full base release path, all matching releases and project sub-path base_path = os.path.join(install_path, release) if not os.path.isdir(base_path): logger.error( "Directory {0} is missing - cannot find CMake package data".format( base_path)) sys.exit(1) logger.info("Using base path for release {0} of {1}".format( release, base_path)) sample_project = find_cmake_sample_project(base_path) if not sample_project: logger.error( "Could not find any sample project from {0} - cannot find CMake package data" .format(base_path)) sys.exit(1) logger.debug("Found build project {0} to build architecture with".format( sample_project)) if nightly: releases = [nightly] if not os.path.isdir( os.path.join(os.path.join(base_path, sample_project, nightly))): logger.error("Could not find release {0} - " "cannot find CMake package data".format( os.path.join(base_path, sample_project, nightly))) sys.exit(1) else: releases = [ d for d in os.listdir(os.path.join(base_path, sample_project)) if os.path.isdir(os.path.join(base_path, sample_project, d)) and d.startswith(release) ] if len(releases) == 0: logger.error("Could not find any releases in {0} - " "cannot find CMake package data".format( os.path.join(base_path, sample_project))) sys.exit(1) logger.debug("Found releases: {0}".format(releases)) if arch: project_path = os.path.join("InstallArea", arch) else: if os.path.isdir( os.path.join(base_path, sample_project, releases[0], "InstallArea")): arch = find_best_arch( os.path.join(base_path, sample_project, releases[0], "InstallArea")) project_path = os.path.join("InstallArea", arch) else: arch = find_best_arch( os.path.join(base_path, sample_project, releases[0])) project_path = arch logger.debug("Using build architecture {0}".format(arch)) return base_path, releases, project_path
def main(): parser = argparse.ArgumentParser( description= 'ATLAS tag munger, calculating tag evolution across a releases series') parser.add_argument( 'release', metavar='RELEASE', nargs="+", help= "Files containing tag lists (NICOS format). If a release series/major is given (e.g., 20.1 or 20.1.5) " "the script will search for the base release and all caches to build the tag files in " "a simple way, without worrying about the details of the NICOS tag files and paths (N.B. " "in the rare cases when there is more than one tag file for a release, the last one will " "be used).") parser.add_argument( '--tagdir', default="tagdir", help= "output directory for tag files, each release will generate an entry here (default \"tagdir\")" ) parser.add_argument( '--prefix', help= "Prefix for the name of the release, when the NICOS information is insufficient" ) parser.add_argument( '--nicospath', default="/afs/cern.ch/atlas/software/dist/nightlies/nicos_work/tags/", help="path to NICOS tag files (defaults to usual CERN AFS location)") parser.add_argument( '--analysispkgfilter', action="store_true", help= "Special post processing for the (Ath)AnalysisBase-2.6.X release series, which " "filters tags to be only those which are missing from standard Athena releases" ) parser.add_argument( '--overwrite', action="store_true", default=False, help= "Overwrite any exisitng configuration files (otherwise, just skip over)" ) parser.add_argument('--debug', '--verbose', "-v", action="store_true", help="switch logging into DEBUG mode") args = parser.parse_args() if args.debug: logger.setLevel(logging.DEBUG) # Case when a single bese release is given - we have to expand this if len(args.release) == 1 and re.match(r"(\d+)\.(\d+)(\.(\d+))?$", args.release[0]): nicos_paths = find_nicos_from_base(args.nicospath, args.release[0]) else: nicos_paths = [] for path in args.release: if os.path.exists(path): nicos_paths.append(path) elif os.path.exists(os.path.join(args.nicospath, path)): nicos_paths.append(os.path.join(args.nicospath, path)) else: logger.error( "Path {0} doesn't exist (even after prepending NICOS path)" .format(path)) sys.exit(1) for release in nicos_paths: release_description = parse_release_data(release, args.prefix) release_tags = parse_tag_file(release, args.analysispkgfilter) logger.info("Processing tags for release {0}".format( release_description["name"])) output_file = os.path.join(args.tagdir, release_description["name"]) if args.overwrite or not os.path.exists(output_file): with open(os.path.join(args.tagdir, release_description["name"]), "w") as tag_output: json.dump( { "release": release_description, "tags": release_tags }, tag_output, indent=2) else: logger.debug("Skipped writing to {0} - overwrite is false".format( output_file))
def main(): parser = argparse.ArgumentParser( description='SVN to git migrator, ATLAS style') parser.add_argument('svnroot', metavar='SVNDIR', help="Location of svn repository root") parser.add_argument('gitrepo', metavar='GITDIR', help="Location of git repository") parser.add_argument( 'tagfiles', nargs="+", metavar='TAGFILE', help= "List of release tag content files to process - all tags found in these files will " "be imported (any already imported tags will be skipped)") parser.add_argument( '--targetbranch', default="package", help= "Target git branch for import. Default is the special value 'package' in which " "each package is imported onto its own branch") parser.add_argument( '--svnpath', metavar='PATH', nargs='+', default=[], help="Restrict actions to this list of paths in the SVN tree (use to " "make small scale tests of the import workflow).") parser.add_argument( '--intermediatetags', action="store_true", help= "Import all tags from oldest release tag found, instead of just release tags" ) parser.add_argument( '--processtrunk', action="store_true", help= "Update trunk versions during the import (False by default, the trunk will be skipped)." ) parser.add_argument( '--svncachefile', metavar='FILE', help= "File containing cache of SVN information - default '[gitrepo].svn.metadata'" ) parser.add_argument( '--authorcachefile', metavar='FILE', help= "File containing cache of author name and email information - default '[gitrepo].author.metadata'" ) parser.add_argument( '--importtimingfile', metavar="FILE", help= "File to dump SVN->git import timing information - default '[gitrepo]-timing.json'" ) parser.add_argument( '--svnfilterexceptions', '--sfe', metavar="FILE", help= "File listing path globs to exempt from SVN import filter (lines with '+PATH') or " "to always reject (lines with '-PATH'); default %(default)s. Use NONE to have no exceptions.", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "atlasoffline-exceptions.txt")) parser.add_argument( '--packageveto', metavar="FILE", help="File listing packages that will be skipped completely on import.", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "atlaspackage-exceptions.txt")) parser.add_argument( '--licensefile', metavar="FILE", help="License file to add to source code files (default " "is to add %(default)s license file)", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "cerncopy.txt")) parser.add_argument( '--licenseexceptions', metavar="FILE", help="File listing path globs to exempt from or " "always apply license file to (same format as --svnfilterexceptions)", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "atlaslicense-exceptions.txt")) parser.add_argument( '--uncrustify', metavar="FILE", help="Uncrustify configuration file to use to process C++ " "sources through before git import (by default uncrustify will not be used)" ) parser.add_argument( '--uncrustifyexceptions', metavar="FILE", help="File listing path globs to exempt from or " "always apply uncrustify to (same format as --svnfilterexceptions)", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "atlasuncrustify-exceptions.txt")) parser.add_argument('--debug', '--verbose', "-v", action="store_true", help="Switch logging into DEBUG mode") # Parse and handle initial arguments args = parser.parse_args() if args.debug: logger.setLevel(logging.DEBUG) # Massage default values if not args.svncachefile: args.svncachefile = os.path.basename(args.gitrepo) + ".svn.metadata" if not args.authorcachefile: args.authorcachefile = os.path.basename( args.gitrepo) + ".author.metadata" if not args.importtimingfile: args.importtimingfile = os.path.basename(args.gitrepo) + "-timing.json" # Set svnroot and git repo, get some starting values svnroot = args.svnroot gitrepo = os.path.abspath(args.gitrepo) start_cwd = os.getcwd() start_timestamp_string = time.strftime("%Y%m%dT%H%M.%S") logger.debug("Set SVN root to {0} and git repo to {1}".format( svnroot, gitrepo)) # Load exception globs svn_path_accept, svn_path_reject = load_exceptions_file( args.svnfilterexceptions) # Load package vetos if args.packageveto: package_veto = load_package_veto(args.packageveto) else: package_veto = [] # License file loading if args.licensefile: with open(args.licensefile) as lfh: license_text = [line.rstrip() for line in lfh.readlines()] else: license_text = None if args.licenseexceptions: license_path_accept, license_path_reject = load_exceptions_file( args.licenseexceptions) else: license_path_accept = license_path_reject = [] # Uncrustify exceptions file if args.uncrustify: uncrustify_path_accept, uncrustify_path_reject = load_exceptions_file( args.uncrustifyexceptions) else: uncrustify_path_accept = uncrustify_path_reject = [] ### Main actions start here # Setup the git repository init_git(gitrepo) # Pull current list of tags here, to fast skip any work already done if args.targetbranch != "package": switch_to_branch(args.targetbranch, orphan=True) current_git_tags = get_current_git_tags(gitrepo) os.chdir(start_cwd) ## SVN interactions and reloading state # Decide which svn packages we will import # Note that if we're pulling the packages from a tag diff file, we also get tags # at this point, otherwise the tag list is empty. svn_packages = get_tags(args.tagfiles, args.svnpath) # Add "trunk" packages, if required if args.processtrunk: for package, tags in svn_packages.iteritems(): if "trunk" not in tags: tags.append("trunk") # Initialise SVN and author metadata cache with any stored values svn_metadata_cache = initialise_metadata(args.svncachefile) author_metadata_cache = initialise_metadata(args.authorcachefile) # Prepare package import scan_svn_tags_and_get_metadata(svnroot, svn_packages, svn_metadata_cache, author_metadata_cache, args.intermediatetags, package_veto=package_veto) # Now presistify metadata cache backup_metadata(svn_metadata_cache, start_cwd, args.svncachefile, start_timestamp_string) backup_metadata(author_metadata_cache, start_cwd, args.authorcachefile, start_timestamp_string) # Setup dictionary for keying by SVN revision number svn_cache_revision_dict = svn_cache_revision_dict_init(svn_metadata_cache) ## git processing actions # Process each SVN tag in order ordered_revisions = svn_cache_revision_dict.keys() ordered_revisions.sort(cmp=lambda x, y: cmp(int(x), int(y))) logger.info("Will process {0} SVN revisions in total".format( len(ordered_revisions))) counter = 0 processed_tags = 0 timing = [] os.chdir(gitrepo) for rev in ordered_revisions: counter += 1 start = time.time() logger.info("SVN Revsion {0} ({1} of {2})".format( rev, counter, len(ordered_revisions))) for pkg_tag in svn_cache_revision_dict[rev]: if get_flattened_git_tag(pkg_tag["package"], pkg_tag["tag"], rev) in current_git_tags: logger.info("Tag {0} exists already - skipping".format( os.path.join(pkg_tag["package"], pkg_tag["tag"]))) continue if args.targetbranch == "package": switch_to_branch(os.path.basename(pkg_tag["package"]), orphan=True) svn_co_tag_and_commit( svnroot, gitrepo, pkg_tag["package"], pkg_tag["tag"], svn_metadata_cache[os.path.basename( pkg_tag["package"])]["svn"][pkg_tag["tag"]][rev], author_metadata_cache, svn_path_accept=svn_path_accept, svn_path_reject=svn_path_reject, package_veto=package_veto, license_text=license_text, license_path_accept=license_path_accept, license_path_reject=license_path_reject, uncrustify_config=args.uncrustify, uncrustify_path_accept=uncrustify_path_accept, uncrustify_path_reject=uncrustify_path_reject) processed_tags += 1 elapsed = time.time() - start logger.info( "{0} processed in {1}s ({2} packages really processed)".format( counter, elapsed, processed_tags)) timing.append(elapsed) # Last task, clean all empty directories (git does not track these, but they are clutter) check_output_with_retry(("git", "clean", "-f", "-d")) if args.importtimingfile: os.chdir(start_cwd) with open(args.importtimingfile, "w") as time_file: json.dump(timing, time_file)
def main(): parser = argparse.ArgumentParser(description='git branch constructor') parser.add_argument('gitrepo', metavar='GITDIR', help="Location of git repository") parser.add_argument('branchname', help="Git branch name to build") parser.add_argument('tagfiles', metavar="TAGFILE", nargs="+", help="Tag files to use to build git branch from") parser.add_argument('--parentbranch', metavar="BRANCH:COMMIT or BRANCH:@TIMESTAMP", help="If branch does not yet exist, use this BRANCH to make it from at COMMIT " "(otherwise an orphaned branch is created). The syntax BRANCH:@TIMESTAMP will " "find the commit closest to the given TIMESTAMP.") parser.add_argument('--baserelease', metavar="FILE", help="For cache releases, use this tag file as the content of the base release on which " "the release was a cache") parser.add_argument('--svnmetadata', metavar="FILE", help="File with SVN metadata per SVN tag in the git repository. " "By default GITREPO.svn.metadata will be used, if it exists.") parser.add_argument('--authorcachefile', metavar='FILE', help="File containing cache of author name and email information - default '[gitrepo].author.metadata'") parser.add_argument('--skipreleasetag', action="store_true", help="Do not create a git tag for this release, nor skip processing if a release tag " "exists - use this option to add packages to a branch encapsulating an entire " "release series, like 'master'.") parser.add_argument('--onlyforward', action="store_true", help="Process tag files as usual, but never " "downgrade a tag to a previous version. This can be used to reconstruct a master branch " "that only goes forward in revision history (it is very useful for the initial master " "branch constuction). In addition branch series releases that overlap with later releases " "will not be imported so that (again) the master branch does not go backwards in time.") parser.add_argument("--commitdate", choices=["now", "release", "author"], help="Strategy for setting git committer date: now - leave as current time; " "release - set to time of the current release being processed; author - " "set to author date, as found from SVN (default %(default)s)", default = "release") parser.add_argument('--debug', '--verbose', "-v", action="store_true", help="Switch logging into DEBUG mode") parser.add_argument('--dryrun', action="store_true", help="Perform no actions, but print what would happen") # Parse and handle initial arguments args = parser.parse_args() if args.debug: logger.setLevel(logging.DEBUG) gitrepo = os.path.abspath(args.gitrepo) branch = args.branchname # If the onlyforward option is set, then we need to preprocess the list of tag content # files in order to ensure that we never jump across time to a previous release series # when making the master branch. This is because the earlier release series will be # branched off from and the later releases on that series really only make sense # for the branch if args.onlyforward: print args.tagfiles args.tagfiles = backskip_filter(args.tagfiles) print args.tagfiles tag_files = [ os.path.abspath(fname) for fname in args.tagfiles ] # If we have a baserelease tag content, then load that here if args.baserelease: with open(args.baserelease) as br_tags_fh: base_tags = json.load(br_tags_fh) else: base_tags = None # Load SVN metadata cache - this is the fastest way to query the SVN ordering in which tags # were made if not args.svnmetadata and os.access(args.gitrepo + ".svn.metadata", os.R_OK): args.svnmetadata = args.gitrepo + ".svn.metadata" else: logger.error("No SVN metadata cache found - cannot proceed") sys.exit(1) with open(args.svnmetadata) as cache_fh: svn_metadata_cache = json.load(cache_fh) logger.info("Loaded SVN metadata from {0}".format(args.svnmetadata)) # Load author cache info if not args.authorcachefile: args.authorcachefile = args.gitrepo + ".author.metadata" try: with open(args.authorcachefile) as cache_fh: author_metadata_cache = json.load(cache_fh) except OSError: logger.warning("No author metadata found - will proceed without") author_metadata_cache = {} # Main branch reconstruction function branch_builder(gitrepo, args.branchname, tag_files, svn_metadata_cache, author_metadata_cache, parentbranch=args.parentbranch, baserelease=base_tags, skipreleasetag=args.skipreleasetag, dryrun=args.dryrun, only_forward=args.onlyforward, commit_date=args.commitdate)
def branch_builder(gitrepo, branch, tag_files, svn_metadata_cache, author_metadata_cache, parentbranch=None, baserelease=None, skipreleasetag=False, dryrun=False, only_forward=False, commit_date="now"): ## @brief Main branch builder function # @param gitrepo The git repository location # @param branch The git branch to work on # @param tag_files The plain tag content files to process # @param svn_metadata_cache The standard metadata cache from SVN # @param author_metadata_cache Cached author data # @param parentbranch If creating a new branch, this is the BRANCH:COMMIT_ID of where to make the new branch from # @param skipreleasetag If @c True then skip creating git tags for each processed release # @param dryrun If @c True, do nothing except print commands that would have been executed # @param only_forward If @c True then never revert a package to a previous version or import a branch tag # @param commit_date Choice for commit date when building branches # Prepare - chdir and then make sure we are on the correct branch os.chdir(gitrepo) prepare_branch_point(branch, parentbranch) # Main loop starts here, with one pass for each tag file we are processing for tag_file in tag_files: with open(tag_file) as tag_file_fh: release_data = json.load(tag_file_fh) tag_list = get_current_git_tags(gitrepo) current_release_tags = get_current_release_tag_dict(tag_list, branch) # Markers for which packages have been processed logger.info("Processing release {0} ({1} current tags)".format(release_data["release"]["name"], len(current_release_tags))) release_tag = git_release_tag(release_data["release"], branch) if release_tag in tag_list and not skipreleasetag: logger.info("Release tag {0} already made - skipping".format(release_tag)) continue if commit_date == "release": logger.info("Setting committer date to {0:.0f}".format(release_data["release"]["timestamp"])) os.environ["GIT_COMMITTER_DATE"] = "{0:.0f}".format(release_data["release"]["timestamp"]) # Find which packages need updated for this new tag content file import_list, packages_considered = find_packages_for_update(release_data, tag_list, branch, svn_metadata_cache, current_release_tags, only_forward) ## Sort the list of tags to be imported by SVN revision number for a # more or less sensible package by package commit history sorted_import_revisions = import_list.keys() sorted_import_revisions.sort(cmp=lambda x,y: cmp(int(x), int(y))) ## Now loop over all the packages we have to import and update them pkg_processed = 0 for revision in sorted_import_revisions: for pkg_import in import_list[revision]: pkg_processed += 1 do_package_import(pkg_import, svn_metadata_cache, author_metadata_cache, release_name=release_data["release"]["name"], branch=branch, dryrun=dryrun, commit_date=commit_date) logger.info("Processed {0}/{1} revisions".format(pkg_processed, len(import_list))) ## After all packages are updated, look for packages which present in the last # release, but not this one, so they need to be removed new_current_release_tags = get_current_release_tag_dict(tag_list, branch) # Updated package list after upgrade packages_to_remove = [] packages_to_revert = {} for package_name, old_package_state in current_release_tags.iteritems(): if package_name in packages_considered: logger.debug("Package {0} was processed for {1}".format(package_name, release_data["release"]["name"])) continue ## @note We have a package that was not "considered" in the current release. # If we don't have a baserelease then this has been removed, so we zap it. # If there is a baserelease... # ...and this package is not in it, it was in the cache, then was removed, so zap it. # ...and this package is in it, then compare the versions and "revert" to the base # release version if they are different. if baserelease: base_package_version = None for package, base_package_data in baserelease["tags"].iteritems(): if base_package_data["package_name"] == package_name: base_package_version = base_package_data break if base_package_version: if base_package_version["svn_tag"] == old_package_state["svn_tag"]: logger.debug("Package {0} remains at base release version {1}".format(base_package_data["package_name"], base_package_version["svn_tag"])) packages_considered.append(package_name) # Flag we dealt this package else: logger.info("Package {0} was removed from cache - reverting to base " "release version {1}".format(base_package_data["package_name"], base_package_version["svn_tag"])) package_name = base_package_data["package_name"] svn_meta_tag_key = os.path.join("tags", base_package_version["svn_tag"]) svn_revision = svn_metadata_cache[package_name]["svn"][svn_meta_tag_key].keys()[0] git_import_tag = get_flattened_git_tag(package, base_package_version["svn_tag"], svn_revision) packages_to_revert[package_name] = {"package": package, "package_name": os.path.basename(package), "git_import_tag": get_flattened_git_tag(package, base_package_version["svn_tag"], svn_revision), "svn_tag": base_package_version["svn_tag"], "svn_revision": svn_revision, "branch_import_tag": get_flattened_git_tag(package, base_package_version["svn_tag"], svn_revision, branch), "svn_meta_tag_key": svn_meta_tag_key, "current_branch_import_tag": current_release_tags[package_name]["git_tag"]} else: logger.info("Package {0} was removed from the cache and is not in the base release".format(package_name)) packages_to_remove.append(package_name) else: logger.info("Package {0} has been removed from the release".format(package_name)) packages_to_remove.append(package_name) if baserelease: logger.info("{0} packages have been reverted to their base SVN state".format(len(packages_to_revert))) for package_name, revert_data in packages_to_revert.iteritems(): do_package_import(revert_data, svn_metadata_cache, author_metadata_cache, release_name=release_data["release"]["name"], branch=branch, dryrun=dryrun, commit_date=commit_date) logger.info("{0} packages have been removed from the release".format(len(packages_to_remove))) for package in packages_to_remove: logger.info("Removing {0} from {1}".format(package, branch)) if not dryrun: package_path = os.path.join(svn_metadata_cache[package]["path"], package) recursive_delete(package_path) check_output_with_retry(("git", "add", "-A"), dryrun=dryrun) cmd = ["git", "commit", "--allow-empty", "-m", "{0} deleted from {1}".format(package_path, branch)] check_output_with_retry(cmd, dryrun=dryrun) check_output_with_retry(("git", "tag", "-d", current_release_tags[package]["git_tag"]), retries=1, dryrun=dryrun) pkg_processed += 1 ## Now, finally, tag the release as done if not skipreleasetag: if release_data["release"]["nightly"]: check_output_with_retry(("git", "tag", release_tag), retries=1, dryrun=dryrun) else: check_output_with_retry(("git", "tag", release_tag, "-a", "-m", "Tagging release {0}".format(release_data["release"]["name"])), retries=1, dryrun=dryrun) logger.info("Tagged release {0} as {1} " "({2} packages processed)".format(release_data["release"]["name"], release_tag, pkg_processed)) else: logger.info("Processed release {0} (no tag; {1} packages processed)".format(release_data["release"]["name"], pkg_processed))
def do_package_import(pkg_import, svn_metadata_cache, author_metadata_cache, release_name="unknown", branch="unknown", dryrun=False, commit_date="now"): ## @brief Import a package's SVN tag onto the current git branch # updating the corresponding git tags # @param pkg_import package import dictionary (see find_packages_for_update for the # structure) # @param svn_metadata_cache The standard metadata cache from SVN # @param author_metadata_cache Cached author data # @param release_name Name of current release being built (used only for generating log messages) # @param branch Current branch name (used only for generating log messages) # @param dryrun Boolean, if @c true then don't actually act # @param commit_date Choices for setting committer date logger.info("Migrating {0} from {1} to {2} for {3}...".format(pkg_import["package"], pkg_import["current_branch_import_tag"], pkg_import["svn_tag"], release_name)) # Need to wipe out all contents in case files were removed from package if not dryrun: recursive_delete(pkg_import["package"]) check_output_with_retry(("git", "checkout", pkg_import["git_import_tag"], pkg_import["package"]), dryrun=dryrun) # Splat Changelog file - we do not want these on the production branches try: os.remove(os.path.join(pkg_import["package"], "ChangeLog")) except OSError: pass # Done - now commit and tag if logger.level <= logging.DEBUG: cmd = ["git", "status"] logger.debug(check_output_with_retry(cmd)) check_output_with_retry(("git", "add", "-A", pkg_import["package"]), dryrun=dryrun) staged = check_output_with_retry(("git", "diff", "--name-only", "--staged"), dryrun=dryrun) if len(staged) == 0 and (not dryrun): # Nothing staged, so skip doing any commit, but do make the import tag for this branch # so that we don't repeat this step again logger.warning("Package {0} - no changes staged for {1}, " "git tagging and skipping commit".format(pkg_import["package"], release_name)) check_output_with_retry(("git", "tag", pkg_import["branch_import_tag"]), retries=1, dryrun=dryrun) return rev_meta = svn_metadata_cache[pkg_import["package_name"]]["svn"][pkg_import["svn_meta_tag_key"]][pkg_import["svn_revision"]] msg = rev_meta["msg"] if pkg_import["svn_tag"] == "trunk": msg += " (trunk r{0})".format(rev_meta["revision"]) else: msg += " ({0})".format(pkg_import["svn_tag"]) cl_diff = changelog_diff(pkg_import["package"], from_tag="/".join(pkg_import["current_branch_import_tag"].split("/")[1:]) if pkg_import["current_branch_import_tag"] else None, to_tag=pkg_import["git_import_tag"]) if cl_diff: msg += "\n\n" + "\n".join(cl_diff) cmd = ["git", "commit", "-m", msg] author = author_string(rev_meta["author"], author_metadata_cache) cmd.append("--author='{0}'".format(author)) cmd.append("--date={0}".format(rev_meta["date"])) if commit_date == "author": os.environ["GIT_COMMITTER_DATE"] = rev_meta["date"] check_output_with_retry(cmd, retries=1, dryrun=dryrun) if commit_date == "author": del os.environ["GIT_COMMITTER_DATE"] check_output_with_retry(("git", "tag", pkg_import["branch_import_tag"]), retries=1, dryrun=dryrun) if pkg_import["current_branch_import_tag"]: check_output_with_retry(("git", "tag", "-d", pkg_import["current_branch_import_tag"]), retries=1, dryrun=dryrun) logger.info("Committed {0} ({1}) onto {2} for {3}".format(pkg_import["package"], pkg_import["svn_tag"], branch, release_name))
def svn_co_tag_and_commit(svnroot, gitrepo, package, tag, svn_metadata=None, author_metadata_cache=None, branch=None, svn_path_accept=[], svn_path_reject=[], package_veto=[], commit=True, revision=None, license_text=None, license_path_accept=[], license_path_reject=[], uncrustify_config=None, uncrustify_path_accept=[], uncrustify_path_reject=[]): ## @brief Make a temporary space, check out from svn, clean-up, copy and then git commit and tag # @param svnroot Base path to SVN repository # @param gitrepo Path to git repository to import to # @param package Path to package root (in git and svn) # @param tag Package tag to import (i.e., path after base package path) # @param svn_metadata SVN metadata cache # @param author_metadata_cache Author name/email cache # @param branch Git branch to switch to before import # @param svn_path_accept Paths to force import to git # @param svn_path_reject Paths to force reject from the import # @param package_veto List of packages to just plain refuse to handle # @param commit Boolean flag to manage commit (can be set to @c False to only checkout and process) # @param license_text List of strings containing the license text to add (if @c False, then no # license file is added) # @param revision Force SVN revision number (useful for svnpull.py, where # no svn metadata is available) # @param license_path_accept Paths to force include in license file addition # @param license_path_reject Paths to exclude from license file addition # @param uncrustify_config Uncrustify configuration file # @param uncrustify_path_accept Paths to force uncrustify to run on # @param uncrustify_path_reject Paths to exclude from uncrustify if package in package_veto: logger.info("Package {0} is vetoed - skipping import".format(package)) return msg = "Importing SVN path {0}/{1} to {0}".format(package, tag) if svn_metadata and tag == "trunk": msg += " (r{0})".format(svn_metadata["revision"]) logger.info(msg) if branch: logger.info("Switching to branch {0}".format(branch)) switch_to_branch(args.targetbranch) tempdir = tempfile.mkdtemp() full_svn_path = os.path.join(tempdir, package) cmd = ["svn", "checkout"] if revision: cmd.extend(["-r", str(revision)]) elif svn_metadata: cmd.extend(["-r", svn_metadata["revision"]]) cmd.extend([os.path.join(svnroot, package, tag), os.path.join(tempdir, package)]) check_output_with_retry(cmd, retries=1, wait=3) # Clean out directory of things we don't want to import svn_cleanup(full_svn_path, svn_co_root=tempdir, svn_path_accept=svn_path_accept, svn_path_reject=svn_path_reject) # If desired, inject a licence into the source code if license_text: svn_license_injector(full_svn_path, svn_co_root=tempdir, license_text=license_text, license_path_accept=license_path_accept, license_path_reject=license_path_reject) # Pass C++ sources through uncrustify if uncrustify_config: uncrustify_sources(full_svn_path, svn_co_root=tempdir, uncrustify_config=uncrustify_config, uncrustify_path_accept=uncrustify_path_accept, uncrustify_path_reject=uncrustify_path_reject) # Copy to git full_git_path = os.path.join(gitrepo, package) package_root, package_name = os.path.split(full_git_path) try: if os.path.isdir(full_git_path): shutil.rmtree(full_git_path, ignore_errors=True) os.makedirs(package_root) except OSError: pass shutil.move(full_svn_path, package_root) if commit: # get ChangeLog diff cl_diff = changelog_diff(package) # Commit check_output_with_retry(("git", "add", "-A", package)) if logger.level <= logging.DEBUG: logger.debug(check_output_with_retry(("git", "status"))) cmd = ["git", "commit", "--allow-empty", "-m", "{0} ({1} - r{2})". format(svn_metadata['msg'], tag.replace('tags/','',1), svn_metadata['revision'])] if svn_metadata: cmd.extend(("--author='{0}'".format(author_string(svn_metadata["author"], author_metadata_cache)), "--date={0}".format(svn_metadata["date"]))) if cl_diff: cmd.extend(("-m", "Diff in ChangeLog:\n" + '\n'.join(cl_diff))) check_output_with_retry(cmd) cmd = ["git", "tag", "-a", get_flattened_git_tag(package, tag, svn_metadata["revision"]), "-m", ""] check_output_with_retry(cmd) # Clean up shutil.rmtree(tempdir)
def svn_co_tag_and_commit(svnroot, gitrepo, package, tag, full_clobber=True, svn_path_accept=[], svn_path_reject=[], revision=None, license_text=None, license_path_accept=[], license_path_reject=[]): ## @brief Make a temporary space, check out from svn, clean-up and copy into git checkout # @param svnroot Base path to SVN repository # @param gitrepo Path to git repository to import to # @param package Path to package root (in git and svn) # @param tag Package tag to import (i.e., path after base package path) # @param full_clobber If @c True then all current files are deleted, if false then # only newly imported files are copied to checkout # @param svn_path_accept Paths to force import to git # @param svn_path_reject Paths to force reject from the import # @param license_text List of strings containing the license text to add (if @c False, then no # license file is added) # @param revision Force SVN revision number # @param license_path_accept Paths to force include in license file addition # @param license_path_reject Paths to exclude from license file addition msg = "Importing SVN path {0}/{1} to {2}/{0}".format(package, tag, gitrepo) logger.info(msg) tempdir = tempfile.mkdtemp() full_svn_path = os.path.join(tempdir, package) cmd = ["svn", "checkout"] if revision: cmd.extend(["-r", str(revision)]) cmd.extend( [os.path.join(svnroot, package, tag), os.path.join(tempdir, package)]) check_output_with_retry(cmd, retries=1, wait=3) # Clean out directory of things we don't want to import svn_cleanup(full_svn_path, svn_co_root=tempdir, svn_path_accept=svn_path_accept, svn_path_reject=svn_path_reject) # If desired, inject a licence into the source code if license_text: svn_license_injector(full_svn_path, svn_co_root=tempdir, license_text=license_text, license_path_accept=license_path_accept, license_path_reject=license_path_reject) # Copy to git full_git_path = os.path.join(gitrepo, package) package_root, package_name = os.path.split(full_git_path) if full_clobber: try: # We need to be a little more sophisticated here, # as the doxygen change on the master branch of # git rewrote docs/mainpage.h to docs/packagedoc.h # So we do need to keep that file as least, if it is # not in the SVN pull area pkgdoc = os.path.join(package, "doc", "packagedoc.h") if os.access(pkgdoc, os.R_OK): dest_dir = os.path.join(full_svn_path, "doc") if not os.access(os.path.join(dest_dir, "packagedoc.h"), os.R_OK): try: os.makedirs(os.path.join(full_svn_path, "doc")) except OSError: pass shutil.copy2(pkgdoc, dest_dir) logger.info( "Doxygen file packagedoc.h was backed up before overwrite" ) shutil.rmtree(full_git_path, ignore_errors=True) os.makedirs(package_root) except OSError: pass logger.info("Replacing complete current git checkout with {0}".format( os.path.join(package, tag))) shutil.move(full_svn_path, package_root) else: for root, dirs, files in os.walk(full_svn_path): for name in files: src_filename = os.path.join(root, name) dst_filename = os.path.join(src_filename[len(tempdir) + 1:]) logger.info( "Pulling {0} into git checkout".format(dst_filename)) if not os.access(os.path.basename(dst_filename), os.R_OK): os.makedirs(os.path.basename(dst_filename)) shutil.copy2(src_filename, dst_filename) # Clean up shutil.rmtree(tempdir)
sys.exit(1) base_path, releases, project_path = find_cmake_releases( args.installpath, args.release, nightly=args.nightly, arch=args.arch) for release in releases: release_description = get_cmake_release_data(base_path, args.release, release, project_path, nightly=args.nightly) logger.debug("Release {0} parsed as {1}/PROJECT/{2}".format( base_path, release, project_path)) release_tags = find_cmake_tags(base_path, release, project_path) output_file = os.path.join(args.tagdir, release_description["name"]) if args.overwrite or not os.path.exists(output_file): with open(os.path.join(args.tagdir, release_description["name"]), "w") as tag_output: my_release_data = { "release": release_description, "tags": release_tags } json.dump(my_release_data, tag_output, indent=2) logger.info("Wrote {0}".format(output_file)) else: logger.debug("Skipped writing to {0} - overwrite is false".format( output_file)) if __name__ == '__main__': main()