def apply_changes(path):
print(f'RUNNING: git diff HEAD..{commit} -- {path}')
diff = git_cmd(f'diff HEAD..{commit} -- {path}')
if len(diff.strip()) > 0:
print('RUNNING: git apply diff.patch')
diff_filename = 'diff.patch'
with open(diff_filename, 'w') as f:
f.write(diff)
git_cmd(f'apply {diff_filename}')
os.remove(diff_filename)
git_cmd(f'add {path}', working_dir)
return True
return False
def update_green_project_revisions(editor_versions_file, project_versions_file,
track, green_revision_jobs, job_id, api_key,
working_dir):
"""Updates green project revisions file for given track. If any updates present, adds to git and returns True. If not, returns False."""
# get the revisions used for the job, the last green project revisions, and Yamato dependency tree
updated_at = str(datetime.datetime.utcnow())
revisions_key = f"{track}_latest_internal" if track == "trunk" else f"{track}_staging"
revisions_key = revisions_key.replace('.', '_')
revisions = load_yml(
editor_versions_file)["editor_versions"][revisions_key]
last_green_job_revisions = load_yml(project_versions_file)
dependency_tree = get_yamato_dependency_tree(job_id, api_key)
if not dependency_tree:
return False
# update revisions for each project
is_updated = False
for job_name in green_revision_jobs:
jobs = [
node for node in dependency_tree["nodes"]
if node["name"].lower() == job_name.lower()
]
if len(jobs) == 0:
print(f'Skipped "{job_name}" [not found in dependency tree]')
continue
job = jobs[0]
if job["status"] == 'success':
print(f'Updating "{job_name}" [job status: {job["status"]}]')
job_name = job_name.replace(' ', '_').lower()
if not last_green_job_revisions.get(job_name):
last_green_job_revisions[job_name] = {}
last_green_job_revisions[job_name]["updated_at"] = updated_at
last_green_job_revisions[job_name][
"last_green_revisions"] = ordereddict_to_dict(revisions)
is_updated = True
else:
print(f'Skipped "{job_name}" [job status: {job["status"]}]')
if is_updated: # at least one project got updated
last_green_job_revisions = ordereddict_to_dict(
last_green_job_revisions)
with open(project_versions_file, 'w') as f:
yaml.dump(last_green_job_revisions, f)
git_cmd(f'add {project_versions_file}', working_dir)
return True
return False
def main(argv):
args = parse_args(argv)
config = load_yml(DEFAULT_CONFIG_FILE)
print(
f'INFO: Updating editor revisions to the latest found using unity-downloader-cli'
)
print(f'INFO: Configuration file: {DEFAULT_CONFIG_FILE}')
ROOT = os.path.abspath(
git_cmd('rev-parse --show-toplevel', cwd='.').strip())
print(f'INFO: Running in {os.path.abspath(os.curdir)}')
try:
editor_version_files = create_version_files(config, ROOT)
if not args.local and len(editor_version_files) > 0:
checkout_and_push(editor_version_files, args.target_branch, ROOT,
'Updating pinned editor revisions')
print(f'INFO: Done updating editor versions.')
return 0
except subprocess.CalledProcessError as err:
print(
f"ERROR: Failed to run '{err.cmd}'\nStdout:\n{err.stdout}\nStderr:\n{err.stderr}"
)
return 1
def main(argv):
logging.basicConfig(level=logging.INFO, format='[%(levelname)s] %(message)s')
args = parse_args(argv)
config = load_yml(DEFAULT_CONFIG_FILE)
shared = load_yml(DEFAULT_SHARED_FILE)
editor_versions_file = config['editor_versions_file'].replace('TRACK',str(args.track))
green_revisions_file = config['green_revisions_file'].replace('TRACK',str(args.track))
yamato_branch = shared['target_branch']
yamato_project_id = config['yamato_project_id']
yamato_nightly_job_definition = config['nightly_job_definition']
try:
working_dir = os.path.abspath(git_cmd('rev-parse --show-toplevel', cwd='.').strip())
print(f'Working directory: {working_dir}')
if args.local:
logging.warning('\n\n!! DEVELOPMENT MODE: will not switch branch, pull or push !!\n')
else:
checkout_and_pull_branch(args.target_branch, working_dir, args.local)
nightly_job_id = get_last_nightly_id(args.apikey, yamato_project_id, yamato_branch, yamato_nightly_job_definition)
print(f'Updating green project revisions according to job {nightly_job_id}.')
if nightly_job_id:
if update_green_project_revisions(editor_versions_file, green_revisions_file, str(args.track), config['green_revision_jobs'], nightly_job_id, args.apikey, working_dir):
commit_and_push(f'[CI] [{str(args.track)}] Updated green project revisions', working_dir, args.local)
else:
print('No projects to update. Exiting successfully without any commit/push.')
return 0
except subprocess.CalledProcessError as err:
logging.error(f"Failed to run '{err.cmd}'\nStdout:\n{err.stdout}\nStderr:\n{err.stderr}")
return 1
def verify_changed_files(editor_versions_file, commit_hash, working_dir):
cmd = ['show', '--pretty=format:', '--name-only', commit_hash]
filenames = git_cmd(cmd, working_dir).strip().replace('\\', '/').split()
assert editor_versions_file in filenames, f'Cannot find {editor_versions_file} in {filenames}'
filenames.remove(editor_versions_file)
assert all(
filename.startswith(EXPECTATIONS_PATH) or filename.endswith('.yml')
for filename in filenames
), (f'Found other files than {editor_versions_file}, .yml, and expectation files in {filenames}'
)
def main(argv):
args = parse_args(argv)
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO,
format='[%(levelname)s] %(message)s')
config = load_config(args.config)
print(
f'INFO: Updating editor revisions to the latest found using unity-downloader-cli'
)
print(f'INFO: Configuration file: {args.config}')
ROOT = os.path.abspath(
git_cmd('rev-parse --show-toplevel', cwd='.').strip())
print(f'INFO: Running in {os.path.abspath(os.curdir)}')
# projectversion_filename = os.path.join(ROOT, config['project_version_file'])
# assert os.path.isfile(projectversion_filename), f'Cannot find {projectversion_filename}'
try:
editor_versions_filename = config['editor_versions_file']
editor_versions_file = load_latest_versions_metafile(
editor_versions_filename)
versions = get_versions_from_unity_downloader(
config['editor_tracks'], config['trunk_track'],
config['unity_downloader_components'], editor_versions_file)
print(f'INFO: Saving {editor_versions_filename}.')
write_versions_file(os.path.join(ROOT, editor_versions_filename),
config['versions_file_header'], versions)
if versions_file_is_unchanged(editor_versions_filename, ROOT):
print(f'INFO: No changes in the versions file, exiting')
else:
subprocess.call(['python', config['ruamel_build_file']])
if args.yamato_parser:
print(
f'INFO: Running {args.yamato_parser} to generate unfolded Yamato YAML...'
)
run_cmd(args.yamato_parser, cwd=ROOT)
if not args.local:
checkout_and_push(editor_versions_filename,
config['yml_files_path'], args.target_branch,
ROOT, args.force_push,
'Updating pinned editor revisions')
print(f'INFO: Done updating editor versions.')
return 0
except subprocess.CalledProcessError as err:
print(
f"ERROR: Failed to run '{err.cmd}'\nStdout:\n{err.stdout}\nStderr:\n{err.stderr}"
)
return 1
def main(argv):
logging.basicConfig(level=logging.INFO,
format='[%(levelname)s] %(message)s')
args = parse_args(argv)
config = load_config(args.config)
editor_versions_file = config['editor_versions_file']
try:
working_dir = args.working_dir or os.path.abspath(
git_cmd('rev-parse --show-toplevel', cwd='.').strip())
if args.local:
logging.warning(
'\n\n!! DEVELOPMENT MODE: will not switch branch, pull or push !!\n'
)
logging.info(f'Working directory: {working_dir}')
verify_changed_files(editor_versions_file, args.revision, working_dir)
if not args.local:
checkout_and_pull_branch(args.target_branch, working_dir)
if git_cmd('rev-parse HEAD').strip() == args.revision:
logging.info(
'No changes compared to current revision. Exiting...')
return 0
if apply_target_revision_changes(editor_versions_file,
config['yml_files_path'],
args.revision, working_dir):
if args.yamato_parser:
regenerate_expectations(args.yamato_parser, working_dir)
commit_msg = get_commit_message(args.revision)
commit_and_push(commit_msg, working_dir, args.local)
else:
logging.info(
'No revision changes to merge. Exiting successfully without any '
'commit/push.')
return 0
except subprocess.CalledProcessError as err:
logging.error(
f"Failed to run '{err.cmd}'\nStdout:\n{err.stdout}\nStderr:\n{err.stderr}"
)
return 1
def checkout_and_push(editor_versions_files, target_branch, root,
commit_message_details):
original_branch = get_current_branch()
git_cmd(f'checkout -B {target_branch}', cwd=root)
for editor_versions_file in editor_versions_files:
git_cmd(f'add {editor_versions_file}', cwd=root)
cmd = [
'commit', '-m',
f'[CI] Updated pinned editor versions \n\n{commit_message_details}'
]
git_cmd(cmd, cwd=root)
cmd = ['push', '--set-upstream', 'origin', target_branch]
assert not (target_branch
in ('master', '2021.1/staging', '10.x.x/release',
'8.x.x/release', '7.x.x/release')), (
'Error: not allowed to force push to {target_branch}.')
cmd.append('--force')
git_cmd(cmd, cwd=root)
git_cmd(f'checkout {original_branch}', cwd=root)
def main(argv):
args = parse_args(argv)
config = load_yml(DEFAULT_CONFIG_FILE)
ROOT = os.path.abspath(
git_cmd('rev-parse --show-toplevel', cwd='.').strip())
print(
f'INFO: Updating editor revisions to the latest found using unity-downloader-cli'
)
print(f'INFO: Configuration file: {DEFAULT_CONFIG_FILE}')
print(f'INFO: Running in {os.path.abspath(os.curdir)}')
try:
current_branch = git_cmd("rev-parse --abbrev-ref HEAD").strip()
print(f'INFO: Running on branch: {current_branch}')
editor_version_files = create_version_files(config, ROOT)
if args.commit_and_push and len(editor_version_files) > 0:
print(f'INFO: Committing and pushing to branch.')
git_cmd(['add', '.'], cwd=ROOT)
git_cmd(['commit', '-m', f'[CI] Updated pinned editor versions'],
cwd=ROOT)
git_cmd(['pull'], cwd=ROOT)
git_cmd(['push'], cwd=ROOT)
else:
print(
f'INFO: Will not commit or push to current branch. Use --commit-and-push to do so.'
)
print(f'INFO: Done updating editor versions.')
return 0
except subprocess.CalledProcessError as err:
print(
f"ERROR: Failed to run '{err.cmd}'\nStdout:\n{err.stdout}\nStderr:\n{err.stderr}"
)
return 1
def commit_and_push(commit_msg, working_dir, track, development_mode=False):
commit_msg = f'{commit_msg}'
if not development_mode:
git_cmd([
'commit', '-m',
f'[CI] [{str(track)}] Updated latest editors metafile'
], working_dir)
#git_cmd(['commit', '-m', f'[{str(track)}] {commit_msg}'], working_dir)
git_cmd('pull', working_dir)
git_cmd('push', working_dir)
def main(argv):
args = parse_args(argv)
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO,
format='[%(levelname)s] %(message)s')
config = load_config(args.config)
print(
f'INFO: Updating editor revisions to the latest found using unity-downloader-cli'
)
print(f'INFO: Configuration file: {args.config}')
ROOT = os.path.abspath(
git_cmd('rev-parse --show-toplevel', cwd='.').strip())
print(f'INFO: Running in {os.path.abspath(os.curdir)}')
# projectversion_filename = os.path.join(ROOT, config['project_version_file'])
# assert os.path.isfile(projectversion_filename), f'Cannot find {projectversion_filename}'
try:
editor_version_files = create_version_files(config, ROOT)
#subprocess.call(['python', config['ruamel_build_file']])
if not args.local and len(editor_version_files) > 0:
checkout_and_push(editor_version_files, config['yml_files_path'],
args.target_branch, ROOT, args.force_push,
'Updating pinned editor revisions')
print(f'INFO: Done updating editor versions.')
return 0
except subprocess.CalledProcessError as err:
print(
f"ERROR: Failed to run '{err.cmd}'\nStdout:\n{err.stdout}\nStderr:\n{err.stderr}"
)
return 1
def commit_and_push(commit_msg, working_dir, development_mode=False):
if not development_mode:
git_cmd(['commit', '-m', commit_msg], working_dir)
git_cmd('pull', working_dir)
git_cmd('push', working_dir)
def checkout_and_pull_branch(branch, working_dir, development_mode=False):
if not development_mode:
git_cmd(f'checkout {branch}', working_dir)
git_cmd('pull', working_dir)
def main(argv):
# initialize files etc.
args = parse_args(argv)
root = os.path.abspath(git_cmd('rev-parse --show-toplevel', cwd='.').strip())
config = load_yml(DEFAULT_CONFIG_FILE)
shared = load_yml(DEFAULT_SHARED_FILE)
editor_versions_file_path = os.path.join(root, config['editor_versions_file'].replace('TRACK',str(args.track)))
if not os.path.exists(editor_versions_file_path):
open(editor_versions_file_path, 'a').close()
track_key = str(args.track).replace('.','_')
current_branch = git_cmd("rev-parse --abbrev-ref HEAD").strip()
# fetch last revision in the file, or None if revision missing
editor_versions_file = load_yml(editor_versions_file_path)
if editor_versions_file.get(track_key):
last_retrieved_revision = editor_versions_file[track_key]["changeset"]["id"] # TODO parse last revision
else:
last_retrieved_revision = None
try:
# fetch all ono revisions up until the last revision in the file
last_revisions_nodes = get_last_revisions_from_ono(args.api_key, last_retrieved_revision, args.ono_branch)
if len(last_revisions_nodes) == 0:
print(f'INFO: No revisions to update.')
return 0
if args.commit_and_push:
print(f'INFO: Pulling branch: {current_branch}).')
git_cmd(f'checkout {current_branch}', root)
git_cmd(f'pull', root)
print(f'INFO: Committing and pushing each revision ({len(last_revisions_nodes)}).')
for revision_node in last_revisions_nodes:
update_revision_file(editor_versions_file_path, revision_node, track_key, args.ono_branch)
git_cmd(['add','.'], cwd=root)
git_cmd(['commit', '-m', f'[CI] [{args.track}] Updated editor to {revision_node["id"]}'], cwd=root)
git_cmd(['pull'], cwd=root)
git_cmd(['push'], cwd=root)
else:
print(f'INFO: Updating the file to most recent revision, but will not git add/commit/push. Use --commit-and-push to do so.')
update_revision_file(editor_versions_file_path, last_revisions_nodes[-1], track_key, args.ono_branch)
print(f'INFO: Done updating editor versions.')
return 0
except subprocess.CalledProcessError as err:
print(f"ERROR: Failed to run '{err.cmd}'\nStdout:\n{err.stdout}\nStderr:\n{err.stderr}")
return 1
import argparse
import os
import subprocess
import sys
import requests
import datetime
import ruamel.yaml
import json
from util.subprocess_helpers import git_cmd, run_cmd
from util.utils import load_yml, ordereddict_to_dict
yaml = ruamel.yaml.YAML()
UPDATED_AT = str(datetime.datetime.utcnow())
DEFAULT_CONFIG_FILE = os.path.join(os.path.abspath(git_cmd('rev-parse --show-toplevel', cwd='.').strip()),'.yamato','config','_editor.metafile')
DEFAULT_SHARED_FILE = os.path.join(os.path.abspath(git_cmd('rev-parse --show-toplevel', cwd='.').strip()),'.yamato','config','__shared.metafile')
# Fetches all changesets from Ono up to the one already present in the _latest_editor_versions_{track}.metafile, and updates _latest_editor_versions_{track}.metafile
# See args in parse_args()
#
#
# Run locally without any git commands (just updates the file locally to most recent retrieved revisions)
# python .\script\editor_scripts\update_revisions_ono.py --track {track} --ono-branch {branch name} --api-key {apikey}
#
# Run in CI (or locally) to make a commit for each retrieved revision (up to the already stored one), from older to newer
# python .\script\editor_scripts\update_revisions_ono.py --track {track} --ono-branch {branch name} --api-key {apikey} --commit-and-push
def parse_args(flags):
parser = argparse.ArgumentParser()
parser.add_argument('--commit-and-push', action='store_true',
help='If specified: commit/push the each revision separately to the current branch. If not specified: only update the file locally to the most recent revision (no git involved')
def versions_file_is_unchanged(file_path, root):
diff = git_cmd(f'diff {file_path}', cwd=root).strip()
return len(diff) == 0
def checkout_and_push(editor_versions_files, yml_files_path, target_branch,
root, force_push, commit_message_details):
original_branch = get_current_branch()
git_cmd(f'checkout -B {target_branch}', cwd=root)
for editor_versions_file in editor_versions_files:
git_cmd(f'add {editor_versions_file}', cwd=root)
git_cmd(f'add {yml_files_path}', cwd=root)
# Expectations generated if yamato-parser is used:
expectations_dir = os.path.join(root, EXPECTATIONS_PATH)
if os.path.isdir(expectations_dir):
git_cmd(f'add {expectations_dir}', cwd=root)
cmd = [
'commit', '-m',
f'[CI] Updated pinned editor versions \n\n{commit_message_details}'
]
git_cmd(cmd, cwd=root)
cmd = ['push', '--set-upstream', 'origin', target_branch]
if force_push:
assert not (target_branch in (
'master', '9.x.x/release', '8.x.x/release', '7.x.x/release')), (
'Error: not allowed to force push to {target_branch}.')
cmd.append('--force')
git_cmd(cmd, cwd=root)
git_cmd(f'checkout {original_branch}', cwd=root)
def get_current_branch():
return git_cmd("rev-parse --abbrev-ref HEAD").strip()
import yaml
import datetime
from util.subprocess_helpers import run_cmd, git_cmd
# These are by convention how the different revisions are categorized.
# These should not be changed unless also updated in Yamato YAML.
SUPPORTED_VERSION_TYPES = ('latest_internal', 'latest_public', 'staging')
PROJECT_VERSION_NAME = 'project_revision'
PLATFORMS = ('windows', 'macos', 'linux', 'android', 'ios')
UPDATED_AT = str(datetime.datetime.utcnow())
SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
# DEFAULT_CONFIG_FILE = os.path.join(SCRIPT_DIR, 'config.yml')
DEFAULT_CONFIG_FILE = os.path.join(
os.path.abspath(git_cmd('rev-parse --show-toplevel', cwd='.').strip()),
'.yamato', 'config', '_editor.metafile')
EXPECTATIONS_PATH = os.path.join('.yamato', 'expectations')
INVALID_VERSION_ERROR = 'Are you sure this is actually a valid unity version?'
VERSION_PARSER_RE = re.compile(
r'Grabbing unity release ([0-9\.a-z]+) which is revision')
def generate_downloader_cmd(track, version, trunk_track, platform,
unity_downloader_components):
"""Generate a list of commmand arguments for the invovation of the unity-downloader-cli."""
assert platform in PLATFORMS, f'Unsupported platform: {platform}'
if version == 'staging':
# --fast avoids problems with ongoing builds. If we hit such it will
# return an older build instead, which is fine for us for this tool.
def get_commit_message(git_hash):
return git_cmd(f'log --format=%B -n 1 {git_hash}')
def regenerate_expectations(yamato_parser, working_dir):
logging.info(
f'Running {yamato_parser} to generate unfolded Yamato YAML...')
run_cmd(yamato_parser, cwd=working_dir)
git_cmd(f'add {EXPECTATIONS_PATH}', working_dir)
def checkout_and_pull_branch(branch, working_dir):
git_cmd(f'checkout {branch}', working_dir)
git_cmd('pull --ff-only', working_dir)
