def update_firmware_docs():
# module can be started from two different locations
if Path.cwd().stem == "docs":
workspace_root = Path.cwd().parent
else:
workspace_root = Path.cwd()
all = list(read_manifests(workspace_root))
file_loader = FileSystemLoader(str(workspace_root / "docs/templates"))
env = Environment(
loader=file_loader,
trim_blocks=True, # trim indents
lstrip_blocks=True, # trim left whitespace
)
# Get current git branch name
# git rev-parse --abbrev-ref HEAD
branchname = git_branch()
# Process all template files
for template_file in (workspace_root / "docs/templates").glob("*.j2"):
template = env.get_template(template_file.name)
output = template.render(info_list=all, branch=branchname)
# output to doc folder
md_filename = template_file.with_suffix(".md").name
print(" - updating:", md_filename)
with open(workspace_root / "docs" / md_filename, "w") as md_file:
md_file.write(output)
def upload_new_version(self, target_dir, commit=False, extra_tags=None):
"""Upload a new version and update the version file for the asset."""
version = self.get_next_version()
self._store.upload(self._name,
version,
target_dir,
extra_tags=extra_tags)
def _write_version():
with open(self.version_file, 'w') as f:
f.write(str(version))
subprocess.check_call([utils.GIT, 'add', self.version_file])
with utils.chdir(SKIA_DIR):
if commit:
with utils.git_branch():
_write_version()
subprocess.check_call([
utils.GIT, 'commit', '-m',
'Update %s version' % self._name
])
subprocess.check_call(
[utils.GIT, 'cl', 'upload', '--bypass-hooks'])
else:
_write_version()
def upload_new_version(self, target_dir, commit=False):
"""Upload a new version and update the version file for the asset."""
version = self.get_next_version()
target_dir = os.path.abspath(target_dir)
with utils.tmp_dir():
zip_file = os.path.join(os.getcwd(), '%d.zip' % version)
zip_utils.zip(target_dir, zip_file, blacklist=ZIP_BLACKLIST)
gs_path = GS_PATH_TMPL % (self._gs_subdir, str(version))
self._gs.copy(zip_file, gs_path)
def _write_version():
with open(self.version_file, 'w') as f:
f.write(str(version))
subprocess.check_call([utils.GIT, 'add', self.version_file])
with utils.chdir(SKIA_DIR):
if commit:
with utils.git_branch():
_write_version()
subprocess.check_call([
utils.GIT, 'commit', '-m',
'Update %s version' % self._name
])
subprocess.check_call(
[utils.GIT, 'cl', 'upload', '--bypass-hooks'])
else:
_write_version()
def gen_toolchain(chrome_path, msvs_version, isolate_file):
"""Update the VS toolchain, isolate it, and return the isolated hash."""
with utils.chdir(chrome_path):
subprocess.check_call([utils.GCLIENT, 'sync'])
depot_tools = subprocess.check_output([
'python', os.path.join('build', 'find_depot_tools.py')]).rstrip()
with utils.git_branch():
vs_toolchain_py = os.path.join('build', 'vs_toolchain.py')
env = os.environ.copy()
env['GYP_MSVS_VERSION'] = msvs_version
subprocess.check_call(['python', vs_toolchain_py, 'update'], env=env)
output = subprocess.check_output(['python', vs_toolchain_py,
'get_toolchain_dir'], env=env).rstrip()
src_dir = get_toolchain_dir(output)
# Mock out absolute paths in win_toolchain.json.
win_toolchain_utils.abstract(os.path.join('build', 'win_toolchain.json'),
os.path.dirname(depot_tools))
# Isolate the toolchain. Assumes we're running on Windows, since the above
# would fail otherwise.
isolate_file_dirname = os.path.dirname(isolate_file)
toolchain_relpath = os.path.relpath(src_dir, isolate_file_dirname)
chrome_relpath = os.path.relpath(os.getcwd(), isolate_file_dirname)
depot_tools_relpath = os.path.relpath(depot_tools, isolate_file_dirname)
isolate = os.path.join(
os.curdir, 'tools', 'luci-go', 'win64', 'isolate.exe')
isolate_cmd = [isolate, 'archive', '--quiet',
'--isolate-server', 'https://isolateserver.appspot.com',
'-i', isolate_file,
'-s', 'win_toolchain_%s.isolated' % msvs_version,
'--extra-variable', 'WIN_TOOLCHAIN_DIR=%s' % toolchain_relpath,
'--extra-variable', 'DEPOT_TOOLS_DIR=%s' % depot_tools_relpath,
'--extra-variable', 'CHROME_DIR=%s' % chrome_relpath]
isolate_out = subprocess.check_output(isolate_cmd).rstrip()
return shlex.split(isolate_out)[0]
def update_sdk_file(skia_path, isolated_hash):
"""Edit the android_sdk_hash file, upload a CL."""
with utils.chdir(skia_path):
with utils.git_branch():
hash_file = os.path.join('infra', 'bots', 'android_sdk_hash')
with open(hash_file, 'w') as f:
f.write(isolated_hash)
subprocess.check_call([utils.GIT, 'add', hash_file])
subprocess.check_call([utils.GIT, 'commit', '-m', 'Update Android SDK'])
subprocess.check_call([utils.GIT, 'cl', 'upload', '--bypass-hooks'])
def update_sdk_file(skia_path, isolated_hash):
"""Edit the android_sdk_hash file, upload a CL."""
with utils.chdir(skia_path):
with utils.git_branch():
hash_file = os.path.join('infra', 'bots', 'android_sdk_hash')
with open(hash_file, 'w') as f:
f.write(isolated_hash)
subprocess.check_call([utils.GIT, 'add', hash_file])
subprocess.check_call(
[utils.GIT, 'commit', '-m', 'Update Android SDK'])
subprocess.check_call(
[utils.GIT, 'cl', 'upload', '--bypass-hooks'])
def update_toolchain_file(skia_path, msvs_version, isolated_hash):
"""Edit the win_toolchain_hash file, upload a CL."""
with utils.chdir(skia_path):
with utils.git_branch():
hash_file = os.path.join('infra', 'bots', 'win_toolchain_hash.json')
with open(hash_file) as f:
hashes = json.load(f)
hashes[msvs_version] = isolated_hash
with open(hash_file, 'w') as f:
json.dump(hashes, f, indent=4, sort_keys=True)
subprocess.check_call([utils.GIT, 'add', hash_file])
subprocess.check_call([utils.GIT, 'commit', '-m', 'Update Win toolchain'])
subprocess.check_call([utils.GIT, 'cl', 'upload', '--bypass-hooks'])
def update_toolchain_file(skia_path, msvs_version, isolated_hash):
"""Edit the win_toolchain_hash file, upload a CL."""
with utils.chdir(skia_path):
with utils.git_branch():
hash_file = os.path.join('infra', 'bots',
'win_toolchain_hash.json')
with open(hash_file) as f:
hashes = json.load(f)
hashes[msvs_version] = isolated_hash
with open(hash_file, 'w') as f:
json.dump(hashes, f, indent=4, sort_keys=True)
subprocess.check_call([utils.GIT, 'add', hash_file])
subprocess.check_call(
[utils.GIT, 'commit', '-m', 'Update Win toolchain'])
subprocess.check_call(
[utils.GIT, 'cl', 'upload', '--bypass-hooks'])
def upload_new_version(self, target_dir, commit=False):
"""Upload a new version and update the version file for the asset."""
version = self.get_next_version()
self._store.upload(self._name, version, target_dir)
def _write_version():
with open(self.version_file, "w") as f:
f.write(str(version))
subprocess.check_call([utils.GIT, "add", self.version_file])
with utils.chdir(SKIA_DIR):
if commit:
with utils.git_branch():
_write_version()
subprocess.check_call([utils.GIT, "commit", "-m", "Update %s version" % self._name])
subprocess.check_call([utils.GIT, "cl", "upload", "--bypass-hooks"])
else:
_write_version()
def gen_toolchain(chrome_path, msvs_version, target_dir):
"""Update the VS toolchain and copy it to the target_dir."""
with utils.chdir(os.path.join(chrome_path, 'src')):
subprocess.check_call([utils.GCLIENT, 'sync'])
depot_tools = subprocess.check_output(
['python', os.path.join('build', 'find_depot_tools.py')]).rstrip()
with utils.git_branch():
vs_toolchain_py = os.path.join('build', 'vs_toolchain.py')
env = os.environ.copy()
env['GYP_MSVS_VERSION'] = msvs_version
subprocess.check_call(['python', vs_toolchain_py, 'update'],
env=env)
output = subprocess.check_output(
['python', vs_toolchain_py, 'get_toolchain_dir'],
env=env).rstrip()
src_dir = get_toolchain_dir(output)
# Mock out absolute paths in win_toolchain.json.
win_toolchain_utils.abstract(
os.path.join('build', 'win_toolchain.json'),
os.path.dirname(depot_tools))
# Copy the toolchain files to the target_dir.
build = os.path.join(os.getcwd(), 'build')
dst_build = os.path.join(target_dir, 'src', 'build')
os.makedirs(dst_build)
for f in ('find_depot_tools.py', 'vs_toolchain.py',
'win_toolchain.json'):
shutil.copyfile(os.path.join(build, f),
os.path.join(dst_build, f))
shutil.copytree(
os.path.join(os.getcwd(), 'tools', 'gyp', 'pylib'),
os.path.join(target_dir, 'src', 'tools', 'gyp', 'pylib'))
dst_depot_tools = os.path.join(target_dir, 'depot_tools')
os.makedirs(dst_depot_tools)
for f in ('gclient.py', 'breakpad.py'):
shutil.copyfile(os.path.join(depot_tools, f),
os.path.join(dst_depot_tools, f))
toolchain_dst = os.path.join(target_dir, 'depot_tools',
os.path.relpath(src_dir, depot_tools))
shutil.copytree(src_dir,
toolchain_dst,
ignore=filter_toolchain_files)
def upload_new_version(self, target_dir, commit=False):
"""Upload a new version and update the version file for the asset."""
version = self.get_next_version()
self._store.upload(self._name, version, target_dir)
def _write_version():
with open(self.version_file, 'w') as f:
f.write(str(version))
subprocess.check_call([utils.GIT, 'add', self.version_file])
with utils.chdir(SKIA_DIR):
if commit:
with utils.git_branch():
_write_version()
subprocess.check_call([
utils.GIT, 'commit', '-m', 'Update %s version' % self._name])
subprocess.check_call([utils.GIT, 'cl', 'upload', '--bypass-hooks'])
else:
_write_version()
def gen_toolchain(chrome_path, msvs_version, isolate_file):
"""Update the VS toolchain, isolate it, and return the isolated hash."""
with utils.chdir(chrome_path):
subprocess.check_call([utils.GCLIENT, 'sync'])
depot_tools = subprocess.check_output(
['python', os.path.join('build', 'find_depot_tools.py')]).rstrip()
with utils.git_branch():
vs_toolchain_py = os.path.join('build', 'vs_toolchain.py')
env = os.environ.copy()
env['GYP_MSVS_VERSION'] = msvs_version
subprocess.check_call(['python', vs_toolchain_py, 'update'],
env=env)
output = subprocess.check_output(
['python', vs_toolchain_py, 'get_toolchain_dir'],
env=env).rstrip()
src_dir = get_toolchain_dir(output)
# Mock out absolute paths in win_toolchain.json.
win_toolchain_utils.abstract(
os.path.join('build', 'win_toolchain.json'),
os.path.dirname(depot_tools))
# Isolate the toolchain. Assumes we're running on Windows, since the above
# would fail otherwise.
isolate_file_dirname = os.path.dirname(isolate_file)
toolchain_relpath = os.path.relpath(src_dir, isolate_file_dirname)
chrome_relpath = os.path.relpath(os.getcwd(), isolate_file_dirname)
depot_tools_relpath = os.path.relpath(depot_tools,
isolate_file_dirname)
isolate = os.path.join(os.curdir, 'tools', 'luci-go', 'win64',
'isolate.exe')
isolate_cmd = [
isolate, 'archive', '--quiet', '--isolate-server',
'https://isolateserver.appspot.com', '-i', isolate_file, '-s',
'win_toolchain_%s.isolated' % msvs_version, '--extra-variable',
'WIN_TOOLCHAIN_DIR=%s' % toolchain_relpath, '--extra-variable',
'DEPOT_TOOLS_DIR=%s' % depot_tools_relpath, '--extra-variable',
'CHROME_DIR=%s' % chrome_relpath
]
isolate_out = subprocess.check_output(isolate_cmd).rstrip()
return shlex.split(isolate_out)[0]
def upload_new_version(self, target_dir, commit=False):
"""Upload a new version and update the version file for the asset."""
version = self.get_next_version()
target_dir = os.path.abspath(target_dir)
with utils.tmp_dir():
zip_file = os.path.join(os.getcwd(), '%d.zip' % version)
zip_utils.zip(target_dir, zip_file, blacklist=ZIP_BLACKLIST)
gs_path = GS_PATH_TMPL % (self._gs_subdir, str(version))
self._gs.copy(zip_file, gs_path)
def _write_version():
with open(self.version_file, 'w') as f:
f.write(str(version))
subprocess.check_call([utils.GIT, 'add', self.version_file])
with utils.chdir(SKIA_DIR):
if commit:
with utils.git_branch():
_write_version()
subprocess.check_call([
utils.GIT, 'commit', '-m', 'Update %s version' % self._name])
subprocess.check_call([utils.GIT, 'cl', 'upload', '--bypass-hooks'])
else:
_write_version()
def gen_toolchain(chrome_path, msvs_version, target_dir):
"""Update the VS toolchain and copy it to the target_dir."""
with utils.chdir(os.path.join(chrome_path, 'src')):
subprocess.check_call([utils.GCLIENT, 'sync'])
depot_tools = subprocess.check_output([
'python', os.path.join('build', 'find_depot_tools.py')]).rstrip()
with utils.git_branch():
vs_toolchain_py = os.path.join('build', 'vs_toolchain.py')
env = os.environ.copy()
env['GYP_MSVS_VERSION'] = msvs_version
subprocess.check_call(['python', vs_toolchain_py, 'update'], env=env)
output = subprocess.check_output(['python', vs_toolchain_py,
'get_toolchain_dir'], env=env).rstrip()
src_dir = get_toolchain_dir(output)
# Mock out absolute paths in win_toolchain.json.
win_toolchain_utils.abstract(os.path.join('build', 'win_toolchain.json'),
os.path.dirname(depot_tools))
# Copy the toolchain files to the target_dir.
build = os.path.join(os.getcwd(), 'build')
dst_build = os.path.join(target_dir, 'src', 'build')
os.makedirs(dst_build)
for f in ('find_depot_tools.py', 'vs_toolchain.py', 'win_toolchain.json'):
shutil.copyfile(os.path.join(build, f), os.path.join(dst_build, f))
shutil.copytree(os.path.join(os.getcwd(), 'tools', 'gyp', 'pylib'),
os.path.join(target_dir, 'src', 'tools', 'gyp', 'pylib'))
dst_depot_tools = os.path.join(target_dir, 'depot_tools')
os.makedirs(dst_depot_tools)
for f in ('gclient.py', 'breakpad.py'):
shutil.copyfile(os.path.join(depot_tools, f),
os.path.join(dst_depot_tools, f))
toolchain_dst = os.path.join(
target_dir, 'depot_tools', os.path.relpath(src_dir, depot_tools))
shutil.copytree(src_dir, toolchain_dst, ignore=filter_toolchain_files)
def __call__(self):
options, args = self.parser.parse_args(self.gitify.args[2:])
if not is_svn():
print "This only works on svn checkouts!"
sys.exit(1)
package_name = basename()
svntype = svn_type()
if svntype == 'tags':
print "Can't work on tags!"
sys.exit(1)
elif svntype == 'unrecognized':
print "Unrecognized svn structure!"
sys.exit(1)
if not exists(config.GIT_CACHE + package_name):
print "No git repository found in %s." % config.GIT_CACHE
print "Initiating cloning into cache."
clone()
else:
# if we already have a cached copy, make sure it's up-to-date:
print "Updating existing cache:"
gitify(args=['fetch', package_name])
# get the branch svn is on
remote_branch = svn_branch()
# the following is just convention:
local_branch = "local/%s" % remote_branch
cwd = os.getcwd()
# perform all index updates in the cache to avoid conflicts
os.chdir(config.GIT_CACHE + package_name)
dummy, existing_branches = popen('git branch', False, False)
existing_branches = [b.strip('* ') for b in existing_branches]
if local_branch in existing_branches:
popen('git checkout -f %s' % local_branch, False, False)
else:
popen('git checkout -f -b %s %s' % (local_branch, remote_branch),
False, False)
os.chdir(cwd)
if not exists('.git'):
popen('cp -Rp %s%s/.git .' % (config.GIT_CACHE, package_name),
False, False)
# if the working copy is on another branch, switch:
if local_branch != git_branch():
if local_branch in existing_branches:
popen('git checkout -f %s' % local_branch)
else:
popen('git checkout -b %s' % local_branch)
assert git_branch() == local_branch, (
"Changing branches failed, is on %r but should be on %r" %
(git_branch(), local_branch))
print "Git branch '%s' is now following svn branch '%s':" % (
local_branch, remote_branch)
popen('svn status')
popen('git status')