def apply_changes(path): print(f'RUNNING: git diff HEAD..{commit} -- {path}') diff = git_cmd(f'diff HEAD..{commit} -- {path}') if len(diff.strip()) > 0: print('RUNNING: git apply diff.patch') diff_filename = 'diff.patch' with open(diff_filename, 'w') as f: f.write(diff) git_cmd(f'apply {diff_filename}') os.remove(diff_filename) git_cmd(f'add {path}', working_dir) return True return False
def update_green_project_revisions(editor_versions_file, project_versions_file, track, green_revision_jobs, job_id, api_key, working_dir): """Updates green project revisions file for given track. If any updates present, adds to git and returns True. If not, returns False.""" # get the revisions used for the job, the last green project revisions, and Yamato dependency tree updated_at = str(datetime.datetime.utcnow()) revisions_key = f"{track}_latest_internal" if track == "trunk" else f"{track}_staging" revisions_key = revisions_key.replace('.', '_') revisions = load_yml( editor_versions_file)["editor_versions"][revisions_key] last_green_job_revisions = load_yml(project_versions_file) dependency_tree = get_yamato_dependency_tree(job_id, api_key) if not dependency_tree: return False # update revisions for each project is_updated = False for job_name in green_revision_jobs: jobs = [ node for node in dependency_tree["nodes"] if node["name"].lower() == job_name.lower() ] if len(jobs) == 0: print(f'Skipped "{job_name}" [not found in dependency tree]') continue job = jobs[0] if job["status"] == 'success': print(f'Updating "{job_name}" [job status: {job["status"]}]') job_name = job_name.replace(' ', '_').lower() if not last_green_job_revisions.get(job_name): last_green_job_revisions[job_name] = {} last_green_job_revisions[job_name]["updated_at"] = updated_at last_green_job_revisions[job_name][ "last_green_revisions"] = ordereddict_to_dict(revisions) is_updated = True else: print(f'Skipped "{job_name}" [job status: {job["status"]}]') if is_updated: # at least one project got updated last_green_job_revisions = ordereddict_to_dict( last_green_job_revisions) with open(project_versions_file, 'w') as f: yaml.dump(last_green_job_revisions, f) git_cmd(f'add {project_versions_file}', working_dir) return True return False
def main(argv): args = parse_args(argv) config = load_yml(DEFAULT_CONFIG_FILE) print( f'INFO: Updating editor revisions to the latest found using unity-downloader-cli' ) print(f'INFO: Configuration file: {DEFAULT_CONFIG_FILE}') ROOT = os.path.abspath( git_cmd('rev-parse --show-toplevel', cwd='.').strip()) print(f'INFO: Running in {os.path.abspath(os.curdir)}') try: editor_version_files = create_version_files(config, ROOT) if not args.local and len(editor_version_files) > 0: checkout_and_push(editor_version_files, args.target_branch, ROOT, 'Updating pinned editor revisions') print(f'INFO: Done updating editor versions.') return 0 except subprocess.CalledProcessError as err: print( f"ERROR: Failed to run '{err.cmd}'\nStdout:\n{err.stdout}\nStderr:\n{err.stderr}" ) return 1
def main(argv): logging.basicConfig(level=logging.INFO, format='[%(levelname)s] %(message)s') args = parse_args(argv) config = load_yml(DEFAULT_CONFIG_FILE) shared = load_yml(DEFAULT_SHARED_FILE) editor_versions_file = config['editor_versions_file'].replace('TRACK',str(args.track)) green_revisions_file = config['green_revisions_file'].replace('TRACK',str(args.track)) yamato_branch = shared['target_branch'] yamato_project_id = config['yamato_project_id'] yamato_nightly_job_definition = config['nightly_job_definition'] try: working_dir = os.path.abspath(git_cmd('rev-parse --show-toplevel', cwd='.').strip()) print(f'Working directory: {working_dir}') if args.local: logging.warning('\n\n!! DEVELOPMENT MODE: will not switch branch, pull or push !!\n') else: checkout_and_pull_branch(args.target_branch, working_dir, args.local) nightly_job_id = get_last_nightly_id(args.apikey, yamato_project_id, yamato_branch, yamato_nightly_job_definition) print(f'Updating green project revisions according to job {nightly_job_id}.') if nightly_job_id: if update_green_project_revisions(editor_versions_file, green_revisions_file, str(args.track), config['green_revision_jobs'], nightly_job_id, args.apikey, working_dir): commit_and_push(f'[CI] [{str(args.track)}] Updated green project revisions', working_dir, args.local) else: print('No projects to update. Exiting successfully without any commit/push.') return 0 except subprocess.CalledProcessError as err: logging.error(f"Failed to run '{err.cmd}'\nStdout:\n{err.stdout}\nStderr:\n{err.stderr}") return 1
def verify_changed_files(editor_versions_file, commit_hash, working_dir): cmd = ['show', '--pretty=format:', '--name-only', commit_hash] filenames = git_cmd(cmd, working_dir).strip().replace('\\', '/').split() assert editor_versions_file in filenames, f'Cannot find {editor_versions_file} in {filenames}' filenames.remove(editor_versions_file) assert all( filename.startswith(EXPECTATIONS_PATH) or filename.endswith('.yml') for filename in filenames ), (f'Found other files than {editor_versions_file}, .yml, and expectation files in {filenames}' )
def main(argv): args = parse_args(argv) if args.verbose: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO, format='[%(levelname)s] %(message)s') config = load_config(args.config) print( f'INFO: Updating editor revisions to the latest found using unity-downloader-cli' ) print(f'INFO: Configuration file: {args.config}') ROOT = os.path.abspath( git_cmd('rev-parse --show-toplevel', cwd='.').strip()) print(f'INFO: Running in {os.path.abspath(os.curdir)}') # projectversion_filename = os.path.join(ROOT, config['project_version_file']) # assert os.path.isfile(projectversion_filename), f'Cannot find {projectversion_filename}' try: editor_versions_filename = config['editor_versions_file'] editor_versions_file = load_latest_versions_metafile( editor_versions_filename) versions = get_versions_from_unity_downloader( config['editor_tracks'], config['trunk_track'], config['unity_downloader_components'], editor_versions_file) print(f'INFO: Saving {editor_versions_filename}.') write_versions_file(os.path.join(ROOT, editor_versions_filename), config['versions_file_header'], versions) if versions_file_is_unchanged(editor_versions_filename, ROOT): print(f'INFO: No changes in the versions file, exiting') else: subprocess.call(['python', config['ruamel_build_file']]) if args.yamato_parser: print( f'INFO: Running {args.yamato_parser} to generate unfolded Yamato YAML...' ) run_cmd(args.yamato_parser, cwd=ROOT) if not args.local: checkout_and_push(editor_versions_filename, config['yml_files_path'], args.target_branch, ROOT, args.force_push, 'Updating pinned editor revisions') print(f'INFO: Done updating editor versions.') return 0 except subprocess.CalledProcessError as err: print( f"ERROR: Failed to run '{err.cmd}'\nStdout:\n{err.stdout}\nStderr:\n{err.stderr}" ) return 1
def main(argv): logging.basicConfig(level=logging.INFO, format='[%(levelname)s] %(message)s') args = parse_args(argv) config = load_config(args.config) editor_versions_file = config['editor_versions_file'] try: working_dir = args.working_dir or os.path.abspath( git_cmd('rev-parse --show-toplevel', cwd='.').strip()) if args.local: logging.warning( '\n\n!! DEVELOPMENT MODE: will not switch branch, pull or push !!\n' ) logging.info(f'Working directory: {working_dir}') verify_changed_files(editor_versions_file, args.revision, working_dir) if not args.local: checkout_and_pull_branch(args.target_branch, working_dir) if git_cmd('rev-parse HEAD').strip() == args.revision: logging.info( 'No changes compared to current revision. Exiting...') return 0 if apply_target_revision_changes(editor_versions_file, config['yml_files_path'], args.revision, working_dir): if args.yamato_parser: regenerate_expectations(args.yamato_parser, working_dir) commit_msg = get_commit_message(args.revision) commit_and_push(commit_msg, working_dir, args.local) else: logging.info( 'No revision changes to merge. Exiting successfully without any ' 'commit/push.') return 0 except subprocess.CalledProcessError as err: logging.error( f"Failed to run '{err.cmd}'\nStdout:\n{err.stdout}\nStderr:\n{err.stderr}" ) return 1
def checkout_and_push(editor_versions_files, target_branch, root, commit_message_details): original_branch = get_current_branch() git_cmd(f'checkout -B {target_branch}', cwd=root) for editor_versions_file in editor_versions_files: git_cmd(f'add {editor_versions_file}', cwd=root) cmd = [ 'commit', '-m', f'[CI] Updated pinned editor versions \n\n{commit_message_details}' ] git_cmd(cmd, cwd=root) cmd = ['push', '--set-upstream', 'origin', target_branch] assert not (target_branch in ('master', '2021.1/staging', '10.x.x/release', '8.x.x/release', '7.x.x/release')), ( 'Error: not allowed to force push to {target_branch}.') cmd.append('--force') git_cmd(cmd, cwd=root) git_cmd(f'checkout {original_branch}', cwd=root)
def main(argv): args = parse_args(argv) config = load_yml(DEFAULT_CONFIG_FILE) ROOT = os.path.abspath( git_cmd('rev-parse --show-toplevel', cwd='.').strip()) print( f'INFO: Updating editor revisions to the latest found using unity-downloader-cli' ) print(f'INFO: Configuration file: {DEFAULT_CONFIG_FILE}') print(f'INFO: Running in {os.path.abspath(os.curdir)}') try: current_branch = git_cmd("rev-parse --abbrev-ref HEAD").strip() print(f'INFO: Running on branch: {current_branch}') editor_version_files = create_version_files(config, ROOT) if args.commit_and_push and len(editor_version_files) > 0: print(f'INFO: Committing and pushing to branch.') git_cmd(['add', '.'], cwd=ROOT) git_cmd(['commit', '-m', f'[CI] Updated pinned editor versions'], cwd=ROOT) git_cmd(['pull'], cwd=ROOT) git_cmd(['push'], cwd=ROOT) else: print( f'INFO: Will not commit or push to current branch. Use --commit-and-push to do so.' ) print(f'INFO: Done updating editor versions.') return 0 except subprocess.CalledProcessError as err: print( f"ERROR: Failed to run '{err.cmd}'\nStdout:\n{err.stdout}\nStderr:\n{err.stderr}" ) return 1
def commit_and_push(commit_msg, working_dir, track, development_mode=False): commit_msg = f'{commit_msg}' if not development_mode: git_cmd([ 'commit', '-m', f'[CI] [{str(track)}] Updated latest editors metafile' ], working_dir) #git_cmd(['commit', '-m', f'[{str(track)}] {commit_msg}'], working_dir) git_cmd('pull', working_dir) git_cmd('push', working_dir)
def main(argv): args = parse_args(argv) if args.verbose: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO, format='[%(levelname)s] %(message)s') config = load_config(args.config) print( f'INFO: Updating editor revisions to the latest found using unity-downloader-cli' ) print(f'INFO: Configuration file: {args.config}') ROOT = os.path.abspath( git_cmd('rev-parse --show-toplevel', cwd='.').strip()) print(f'INFO: Running in {os.path.abspath(os.curdir)}') # projectversion_filename = os.path.join(ROOT, config['project_version_file']) # assert os.path.isfile(projectversion_filename), f'Cannot find {projectversion_filename}' try: editor_version_files = create_version_files(config, ROOT) #subprocess.call(['python', config['ruamel_build_file']]) if not args.local and len(editor_version_files) > 0: checkout_and_push(editor_version_files, config['yml_files_path'], args.target_branch, ROOT, args.force_push, 'Updating pinned editor revisions') print(f'INFO: Done updating editor versions.') return 0 except subprocess.CalledProcessError as err: print( f"ERROR: Failed to run '{err.cmd}'\nStdout:\n{err.stdout}\nStderr:\n{err.stderr}" ) return 1
def commit_and_push(commit_msg, working_dir, development_mode=False): if not development_mode: git_cmd(['commit', '-m', commit_msg], working_dir) git_cmd('pull', working_dir) git_cmd('push', working_dir)
def checkout_and_pull_branch(branch, working_dir, development_mode=False): if not development_mode: git_cmd(f'checkout {branch}', working_dir) git_cmd('pull', working_dir)
def main(argv): # initialize files etc. args = parse_args(argv) root = os.path.abspath(git_cmd('rev-parse --show-toplevel', cwd='.').strip()) config = load_yml(DEFAULT_CONFIG_FILE) shared = load_yml(DEFAULT_SHARED_FILE) editor_versions_file_path = os.path.join(root, config['editor_versions_file'].replace('TRACK',str(args.track))) if not os.path.exists(editor_versions_file_path): open(editor_versions_file_path, 'a').close() track_key = str(args.track).replace('.','_') current_branch = git_cmd("rev-parse --abbrev-ref HEAD").strip() # fetch last revision in the file, or None if revision missing editor_versions_file = load_yml(editor_versions_file_path) if editor_versions_file.get(track_key): last_retrieved_revision = editor_versions_file[track_key]["changeset"]["id"] # TODO parse last revision else: last_retrieved_revision = None try: # fetch all ono revisions up until the last revision in the file last_revisions_nodes = get_last_revisions_from_ono(args.api_key, last_retrieved_revision, args.ono_branch) if len(last_revisions_nodes) == 0: print(f'INFO: No revisions to update.') return 0 if args.commit_and_push: print(f'INFO: Pulling branch: {current_branch}).') git_cmd(f'checkout {current_branch}', root) git_cmd(f'pull', root) print(f'INFO: Committing and pushing each revision ({len(last_revisions_nodes)}).') for revision_node in last_revisions_nodes: update_revision_file(editor_versions_file_path, revision_node, track_key, args.ono_branch) git_cmd(['add','.'], cwd=root) git_cmd(['commit', '-m', f'[CI] [{args.track}] Updated editor to {revision_node["id"]}'], cwd=root) git_cmd(['pull'], cwd=root) git_cmd(['push'], cwd=root) else: print(f'INFO: Updating the file to most recent revision, but will not git add/commit/push. Use --commit-and-push to do so.') update_revision_file(editor_versions_file_path, last_revisions_nodes[-1], track_key, args.ono_branch) print(f'INFO: Done updating editor versions.') return 0 except subprocess.CalledProcessError as err: print(f"ERROR: Failed to run '{err.cmd}'\nStdout:\n{err.stdout}\nStderr:\n{err.stderr}") return 1
import argparse import os import subprocess import sys import requests import datetime import ruamel.yaml import json from util.subprocess_helpers import git_cmd, run_cmd from util.utils import load_yml, ordereddict_to_dict yaml = ruamel.yaml.YAML() UPDATED_AT = str(datetime.datetime.utcnow()) DEFAULT_CONFIG_FILE = os.path.join(os.path.abspath(git_cmd('rev-parse --show-toplevel', cwd='.').strip()),'.yamato','config','_editor.metafile') DEFAULT_SHARED_FILE = os.path.join(os.path.abspath(git_cmd('rev-parse --show-toplevel', cwd='.').strip()),'.yamato','config','__shared.metafile') # Fetches all changesets from Ono up to the one already present in the _latest_editor_versions_{track}.metafile, and updates _latest_editor_versions_{track}.metafile # See args in parse_args() # # # Run locally without any git commands (just updates the file locally to most recent retrieved revisions) # python .\script\editor_scripts\update_revisions_ono.py --track {track} --ono-branch {branch name} --api-key {apikey} # # Run in CI (or locally) to make a commit for each retrieved revision (up to the already stored one), from older to newer # python .\script\editor_scripts\update_revisions_ono.py --track {track} --ono-branch {branch name} --api-key {apikey} --commit-and-push def parse_args(flags): parser = argparse.ArgumentParser() parser.add_argument('--commit-and-push', action='store_true', help='If specified: commit/push the each revision separately to the current branch. If not specified: only update the file locally to the most recent revision (no git involved')
def versions_file_is_unchanged(file_path, root): diff = git_cmd(f'diff {file_path}', cwd=root).strip() return len(diff) == 0
def checkout_and_push(editor_versions_files, yml_files_path, target_branch, root, force_push, commit_message_details): original_branch = get_current_branch() git_cmd(f'checkout -B {target_branch}', cwd=root) for editor_versions_file in editor_versions_files: git_cmd(f'add {editor_versions_file}', cwd=root) git_cmd(f'add {yml_files_path}', cwd=root) # Expectations generated if yamato-parser is used: expectations_dir = os.path.join(root, EXPECTATIONS_PATH) if os.path.isdir(expectations_dir): git_cmd(f'add {expectations_dir}', cwd=root) cmd = [ 'commit', '-m', f'[CI] Updated pinned editor versions \n\n{commit_message_details}' ] git_cmd(cmd, cwd=root) cmd = ['push', '--set-upstream', 'origin', target_branch] if force_push: assert not (target_branch in ( 'master', '9.x.x/release', '8.x.x/release', '7.x.x/release')), ( 'Error: not allowed to force push to {target_branch}.') cmd.append('--force') git_cmd(cmd, cwd=root) git_cmd(f'checkout {original_branch}', cwd=root)
def get_current_branch(): return git_cmd("rev-parse --abbrev-ref HEAD").strip()
import yaml import datetime from util.subprocess_helpers import run_cmd, git_cmd # These are by convention how the different revisions are categorized. # These should not be changed unless also updated in Yamato YAML. SUPPORTED_VERSION_TYPES = ('latest_internal', 'latest_public', 'staging') PROJECT_VERSION_NAME = 'project_revision' PLATFORMS = ('windows', 'macos', 'linux', 'android', 'ios') UPDATED_AT = str(datetime.datetime.utcnow()) SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) # DEFAULT_CONFIG_FILE = os.path.join(SCRIPT_DIR, 'config.yml') DEFAULT_CONFIG_FILE = os.path.join( os.path.abspath(git_cmd('rev-parse --show-toplevel', cwd='.').strip()), '.yamato', 'config', '_editor.metafile') EXPECTATIONS_PATH = os.path.join('.yamato', 'expectations') INVALID_VERSION_ERROR = 'Are you sure this is actually a valid unity version?' VERSION_PARSER_RE = re.compile( r'Grabbing unity release ([0-9\.a-z]+) which is revision') def generate_downloader_cmd(track, version, trunk_track, platform, unity_downloader_components): """Generate a list of commmand arguments for the invovation of the unity-downloader-cli.""" assert platform in PLATFORMS, f'Unsupported platform: {platform}' if version == 'staging': # --fast avoids problems with ongoing builds. If we hit such it will # return an older build instead, which is fine for us for this tool.
def get_commit_message(git_hash): return git_cmd(f'log --format=%B -n 1 {git_hash}')
def regenerate_expectations(yamato_parser, working_dir): logging.info( f'Running {yamato_parser} to generate unfolded Yamato YAML...') run_cmd(yamato_parser, cwd=working_dir) git_cmd(f'add {EXPECTATIONS_PATH}', working_dir)
def checkout_and_pull_branch(branch, working_dir): git_cmd(f'checkout {branch}', working_dir) git_cmd('pull --ff-only', working_dir)