def DownloadBaselinesForOnePlatform(baseline_subdir):
"""Download most recently generated baseline images for a single platform,
and add any new ones to SVN control.
@param baseline_subdir
"""
# Create repo_to_modify to handle the SVN repository we will add files to.
gm_dir = os.path.join(os.pardir, GM_EXPECTED_DIR) # Shouldn't assume we're in trunk...
try:
os.makedirs(gm_dir)
except:
pass
repo_to_modify = svn.Svn(gm_dir)
repo_to_modify.Checkout(GetBaselineSvnUrl(baseline_subdir), baseline_subdir)
# If there are any locally modified files in that directory, exit
# (so that we don't risk overwriting the user's previous work).
new_and_modified_files = repo_to_modify.GetNewAndModifiedFiles()
if not options.ignore_local_mods:
if new_and_modified_files:
raise Exception('Exiting because there are already new and/or '
'modified files in %s. To continue in spite of '
'that, run with %s option.' % (
baseline_subdir, OPTION_IGNORE_LOCAL_MODS))
# Download actual gm images into a separate repo in a temporary directory.
actual_dir = tempfile.mkdtemp()
actual_repo = svn.Svn(actual_dir)
print 'Using %s as a temp dir' % actual_dir
actual_url = GetLatestResultsSvnUrl(svn=actual_repo, baseline_subdir=baseline_subdir,
builder_suffix=options.builder_suffix)
print 'Reading actual buildbot GM results from %s' % actual_url
actual_repo.Checkout(actual_url, '.')
# Copy any of those files we are interested in into repo_to_modify,
# and then delete the temporary directory.
CopyMatchingFiles(source_dir=actual_dir,
dest_dir=os.path.join(gm_dir, baseline_subdir),
filename_pattern='*.png',
only_copy_updates=(not options.add_new_files))
shutil.rmtree(actual_dir)
actual_repo = None
# Add any new files to SVN control (if we are running with add_new_files).
if options.add_new_files:
new_files = repo_to_modify.GetNewFiles()
if new_files:
repo_to_modify.AddFiles(sorted(new_files))
# Set the mimetype property on any new/modified image files in
# baseline_subdir. (We used to set the mimetype property on *all* image
# files in the directory, even those whose content wasn't changing,
# but that caused confusion. See
# http://code.google.com/p/skia/issues/detail?id=618 .)
modified_files = repo_to_modify.GetNewAndModifiedFiles()
repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.png')),
svn.PROPERTY_MIMETYPE, 'image/png')
repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.pdf')),
svn.PROPERTY_MIMETYPE, 'application/pdf')
def Main(options, args):
"""Download most recently generated baseline images for a given platform,
and add any new ones to SVN control.
@param options
@param args
"""
num_args = len(args)
if num_args != 1:
RaiseUsageException()
# Create repo_to_modify to handle the SVN repository we will add files to.
baseline_subdir = args[0].rstrip(os.sep)
if not os.path.isdir(baseline_subdir):
raise Exception('could not find baseline_subdir "%s"' %
baseline_subdir)
repo_to_modify = svn.Svn(baseline_subdir)
# If there are any locally modified files in that directory, exit
# (so that we don't risk overwriting the user's previous work).
new_and_modified_files = repo_to_modify.GetNewAndModifiedFiles()
if not options.ignore_local_mods:
if new_and_modified_files:
raise Exception('Exiting because there are already new and/or '
'modified files in %s. To continue in spite of '
'that, run with %s option.' %
(baseline_subdir, OPTION_IGNORE_LOCAL_MODS))
# Download actual gm images into a separate repo in a temporary directory.
tempdir = tempfile.mkdtemp()
download_repo = svn.Svn(tempdir)
download_repo.Checkout(GetLatestResultsSvnUrl(baseline_subdir), '.')
# Copy any of those files we are interested in into repo_to_modify,
# and then delete the temporary directory.
CopyMatchingFiles(source_dir=tempdir,
dest_dir=baseline_subdir,
filename_pattern='*.png',
only_copy_updates=(not options.add_new_files))
shutil.rmtree(tempdir)
download_repo = None
# Add any new files to SVN control (if we are running with add_new_files).
new_files = repo_to_modify.GetNewFiles()
if new_files and options.add_new_files:
repo_to_modify.AddFiles(new_files)
# Set the mimetype property on *all* image files in baseline_subdir, even
# the ones that were already there (in case that property wasn't properly
# set already).
repo_to_modify.SetPropertyByFilenamePattern('*.png', svn.PROPERTY_MIMETYPE,
'image/png')
repo_to_modify.SetPropertyByFilenamePattern('*.pdf', svn.PROPERTY_MIMETYPE,
'application/pdf')
def _create_svn_checkout(dir_path, repo_url):
"""Creates local checkout of an SVN repository at the specified directory
path, returning an svn.Svn object referring to the local checkout.
Args:
dir_path: path to the local checkout; if this directory does not yet exist,
it will be created and the repo will be checked out into it
repo_url: URL of SVN repo to check out into dir_path (unless the local
checkout already exists)
Returns: an svn.Svn object referring to the local checkout.
"""
local_checkout = svn.Svn(dir_path)
if not os.path.isdir(dir_path):
os.makedirs(dir_path)
local_checkout.Checkout(repo_url, '.')
return local_checkout
def CompareBaselines(gm_basedir, path_to_skdiff, svn_gm_url):
"""Compare the gm results within gm_basedir against those already
committed to the Skia repository.
@param gm_basedir
@param path_to_skdiff
@param svn_gm_url base URL of Subversion repository where we store the
expected GM results
"""
# Validate parameters, filling in default values if necessary and possible.
if not os.path.isdir(gm_basedir):
raise Exception('cannot find gm_basedir at %s; maybe you need to '
'specify the %s option?' %
(gm_basedir, OPTION_GM_BASEDIR))
path_to_skdiff = FindPathToSkDiff(path_to_skdiff)
tempdir_base = tempfile.mkdtemp()
# Download all checked-in baseline images to a temp directory
checkedin_dir = os.path.join(tempdir_base, 'checkedin')
os.mkdir(checkedin_dir)
svn.Svn(checkedin_dir).Checkout(svn_gm_url, '.')
# Flatten those checked-in baseline images into checkedin_flattened_dir
checkedin_flattened_dir = os.path.join(tempdir_base, 'checkedin_flattened')
os.mkdir(checkedin_flattened_dir)
Flatten(source_dir=checkedin_dir,
dest_dir=checkedin_flattened_dir,
subdirectory_pattern='base-*')
# Flatten the local baseline images into local_flattened_dir
local_flattened_dir = os.path.join(tempdir_base, 'local_flattened')
os.mkdir(local_flattened_dir)
Flatten(source_dir=gm_basedir,
dest_dir=local_flattened_dir,
subdirectory_pattern='base-*')
# Run skdiff to compare checkedin_flattened_dir against local_flattened_dir
diff_dir = os.path.join(tempdir_base, 'diffs')
os.mkdir(diff_dir)
RunCommand('%s %s %s %s' % (path_to_skdiff, checkedin_flattened_dir,
local_flattened_dir, diff_dir))
print '\nskdiff results are ready in file://%s/index.html' % diff_dir
def SvnDiff(path_to_skdiff, dest_dir):
"""Generates a visual diff of all pending changes in the local SVN checkout.
@param path_to_skdiff
@param dest_dir existing directory within which to write results
"""
# Validate parameters, filling in default values if necessary and possible.
path_to_skdiff = FindPathToSkDiff(path_to_skdiff)
if not dest_dir:
dest_dir = tempfile.mkdtemp()
# Prepare temporary directories.
modified_flattened_dir = os.path.join(dest_dir, 'modified_flattened')
original_flattened_dir = os.path.join(dest_dir, 'original_flattened')
diff_dir = os.path.join(dest_dir, 'diffs')
for dir in [modified_flattened_dir, original_flattened_dir, diff_dir] :
shutil.rmtree(dir, ignore_errors=True)
os.mkdir(dir)
# Get a list of all locally modified (including added/deleted) files,
# descending subdirectories.
svn_repo = svn.Svn('.')
modified_file_paths = svn_repo.GetFilesWithStatus(
svn.STATUS_ADDED | svn.STATUS_DELETED | svn.STATUS_MODIFIED)
# For each modified file:
# 1. copy its current contents into modified_flattened_dir
# 2. copy its original contents into original_flattened_dir
for modified_file_path in modified_file_paths:
dest_filename = re.sub(os.sep, '__', modified_file_path)
# If the file had STATUS_DELETED, it won't exist anymore...
if os.path.isfile(modified_file_path):
shutil.copyfile(modified_file_path,
os.path.join(modified_flattened_dir, dest_filename))
svn_repo.ExportBaseVersionOfFile(
modified_file_path,
os.path.join(original_flattened_dir, dest_filename))
# Run skdiff: compare original_flattened_dir against modified_flattened_dir
RunCommand('%s %s %s %s' % (path_to_skdiff, original_flattened_dir,
modified_flattened_dir, diff_dir))
print '\nskdiff results are ready in file://%s/index.html' % diff_dir
def generate_revision_key(self, url, ud, d):
key = self._revision_key(url, ud, d)
return "%s-%s" % (key, bb.data.getVar("PN", d, True) or "")
import cvs
import git
import local
import svn
import wget
import svk
import ssh
import perforce
import bzr
import hg
import osc
import repo
methods.append(local.Local())
methods.append(wget.Wget())
methods.append(svn.Svn())
methods.append(git.Git())
methods.append(cvs.Cvs())
methods.append(svk.Svk())
methods.append(ssh.SSH())
methods.append(perforce.Perforce())
methods.append(bzr.Bzr())
methods.append(hg.Hg())
methods.append(osc.Osc())
methods.append(repo.Repo())
def SvnDiff(path_to_skdiff, dest_dir, source_dir):
"""Generates a visual diff of all pending changes in source_dir.
@param path_to_skdiff
@param dest_dir existing directory within which to write results
@param source_dir
"""
# Validate parameters, filling in default values if necessary and possible.
path_to_skdiff = os.path.abspath(FindPathToSkDiff(path_to_skdiff))
if not dest_dir:
dest_dir = tempfile.mkdtemp()
dest_dir = os.path.abspath(dest_dir)
os.chdir(source_dir)
svn_repo = svn.Svn('.')
using_svn = True
try:
svn_repo.GetInfo()
except:
using_svn = False
# Prepare temporary directories.
modified_flattened_dir = os.path.join(dest_dir, 'modified_flattened')
original_flattened_dir = os.path.join(dest_dir, 'original_flattened')
diff_dir = os.path.join(dest_dir, 'diffs')
for dir in [modified_flattened_dir, original_flattened_dir, diff_dir]:
shutil.rmtree(dir, ignore_errors=True)
os.mkdir(dir)
# Get a list of all locally modified (including added/deleted) files,
# descending subdirectories.
if using_svn:
modified_file_paths = svn_repo.GetFilesWithStatus(
svn.STATUS_ADDED | svn.STATUS_DELETED | svn.STATUS_MODIFIED)
else:
modified_file_paths = git_utils.GetModifiedFiles()
# For each modified file:
# 1. copy its current contents into modified_flattened_dir
# 2. copy its original contents into original_flattened_dir
for modified_file_path in modified_file_paths:
if modified_file_path.endswith('.json'):
# Special handling for JSON files, in the hopes that they
# contain GM result summaries.
original_file = tempfile.NamedTemporaryFile(delete=False)
original_file.close()
if using_svn:
svn_repo.ExportBaseVersionOfFile(modified_file_path,
original_file.name)
else:
_GitExportBaseVersionOfFile(modified_file_path,
original_file.name)
modified_dir = os.path.dirname(modified_file_path)
platform_prefix = (re.sub(re.escape(os.sep), '__',
os.path.splitdrive(modified_dir)[1]) +
'__')
_CallJsonDiff(old_json_path=original_file.name,
new_json_path=modified_file_path,
old_flattened_dir=original_flattened_dir,
new_flattened_dir=modified_flattened_dir,
filename_prefix=platform_prefix)
os.remove(original_file.name)
else:
dest_filename = re.sub(re.escape(os.sep), '__', modified_file_path)
# If the file had STATUS_DELETED, it won't exist anymore...
if os.path.isfile(modified_file_path):
shutil.copyfile(
modified_file_path,
os.path.join(modified_flattened_dir, dest_filename))
if using_svn:
svn_repo.ExportBaseVersionOfFile(
modified_file_path,
os.path.join(original_flattened_dir, dest_filename))
else:
_GitExportBaseVersionOfFile(
modified_file_path,
os.path.join(original_flattened_dir, dest_filename))
# Run skdiff: compare original_flattened_dir against modified_flattened_dir
RunCommand('%s %s %s %s' % (path_to_skdiff, original_flattened_dir,
modified_flattened_dir, diff_dir))
print '\nskdiff results are ready in file://%s/index.html' % diff_dir
