def CreatePkgFile(self):
"""Create and pkg file for use with the FreeBSD pkg tool.
Create a package from the result of the package's InstallStep.
"""
install_dir = self.GetInstallLocation()
if not os.path.exists(install_dir):
Log('Skiping pkg creation. Install dir not found: %s' %
install_dir)
return
# Strip all elf or pexe files in the install directory (except .o files
# since we don't want to strip, for example, crt1.o)
if not self.config.debug and self.config.toolchain != 'emscripten':
strip = util.GetStrip(self.config)
for root, _, files in os.walk(install_dir):
for filename in files:
fullname = os.path.join(root, filename)
if (os.path.isfile(fullname) and util.IsElfFile(fullname)
and os.path.splitext(fullname)[1] != '.o'):
Log('stripping: %s %s' % (strip, fullname))
subprocess.check_call([strip, fullname])
abi = 'pkg_' + self.config.toolchain
if self.config.arch != self.config.toolchain:
abi += "_" + util.arch_to_pkgarch[self.config.arch]
abi_dir = os.path.join(paths.PUBLISH_ROOT, abi)
pkg_file = os.path.join(abi_dir,
'%s-%s.tbz' % (self.NAME, self.VERSION))
util.Makedirs(abi_dir)
deps = self.DEPENDS
if self.config.toolchain != 'glibc':
deps = []
bsd_pkg.CreatePkgFile(self.NAME, self.VERSION, self.config.arch,
self.GetInstallLocation(), pkg_file, deps)
def DownloadFiles(files, check_hashes=True, parallel=False):
"""Download one of more files to the local disk.
Args:
files: List of FileInfo objects to download.
check_hashes: When False assume local files have the correct
hash otherwise always check the hashes match the onces in the
FileInfo ojects.
Returns:
List of (filename, url) tuples.
"""
files_to_download = []
filenames = []
download_dir = webports.package_index.PREBUILT_ROOT
if not os.path.exists(download_dir):
os.makedirs(download_dir)
for file_info in files:
basename = os.path.basename(file_info.url)
file_info.name = os.path.join(download_dir, basename)
filenames.append((file_info.name, file_info.url))
if os.path.exists(file_info.name):
if not check_hashes or CheckHash(file_info.name, file_info.md5):
Log('Up-to-date: %s' % file_info.name)
continue
files_to_download.append(file_info)
def Check(file_info):
if check_hashes and not CheckHash(file_info.name, file_info.md5):
raise webports.Error(
'Checksum failed: %s\nExpected=%s\nActual=%s' %
(file_info.name, file_info.md5, GetHash(file_info.name)))
if not files_to_download:
Log('All files up-to-date')
else:
total_size = sum(f.size for f in files_to_download)
Log('Need to download %d/%d files [%s]' %
(len(files_to_download), len(files), FormatSize(total_size)))
gsutil = FindGsutil()
if parallel:
remaining_files = files_to_download
num_files = 20
while remaining_files:
files = remaining_files[:num_files]
remaining_files = remaining_files[num_files:]
cmd = gsutil + ['-m', 'cp'] + [f.gsurl for f in files
] + [download_dir]
LogVerbose(cmd)
subprocess.check_call(cmd)
for file_info in files:
Check(file_info)
else:
for file_info in files_to_download:
webports.DownloadFile(file_info.name, file_info.url)
Check(file_info)
return filenames
def Clean(self):
pkg = self.PackageFile()
Log('removing %s' % pkg)
if os.path.exists(pkg):
os.remove(pkg)
stamp_dir = os.path.join(paths.STAMP_DIR, self.NAME)
Log('removing %s' % stamp_dir)
util.RemoveTree(stamp_dir)
def RunGitCmd(directory, cmd, error_ok=False):
cmd = ['git'] + cmd
LogVerbose('%s' % ' '.join(cmd))
p = subprocess.Popen(cmd,
cwd=directory,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
stdout, stderr = p.communicate()
if not error_ok and p.returncode != 0:
if stdout:
Log(stdout)
if stderr:
Log(stderr)
raise Error('git command failed: %s' % cmd)
Trace('git exited with %d' % p.returncode)
return p.returncode
def WriteStamp(stamp_file, stamp_contents):
"""Write a stamp file to disk with the given file contents."""
stamp_dir = os.path.dirname(stamp_file)
util.Makedirs(stamp_dir)
with open(stamp_file, 'w') as f:
f.write(stamp_contents)
Log('Wrote stamp: %s' % stamp_file)
def InitGitRepo(directory):
"""Initialize the source git repository for a given package directory.
This function works for unpacked tar files as well as cloned git
repositories. It sets up an 'upstream' branch pointing and the
pristine upstream sources and a 'master' branch will contain changes
specific to webports (normally the result of applying nacl.patch).
Args:
directory: Directory containing unpacked package sources.
"""
git_dir = os.path.join(directory, '.git')
# If the upstream ref exists then we've already initialized this repo
if os.path.exists(os.path.join(git_dir, 'refs', 'heads', 'upstream')):
return
if os.path.exists(git_dir):
Log('Init existing git repo: %s' % directory)
RunGitCmd(directory, ['checkout', '-b', 'placeholder'])
RunGitCmd(directory, ['branch', '-D', 'upstream'], error_ok=True)
RunGitCmd(directory, ['branch', '-D', 'master'], error_ok=True)
RunGitCmd(directory, ['checkout', '-b', 'upstream'])
RunGitCmd(directory, ['checkout', '-b', 'master'])
RunGitCmd(directory, ['branch', '-D', 'placeholder'])
else:
Log('Init new git repo: %s' % directory)
RunGitCmd(directory, ['init'])
try:
# Setup a bogus identity on the buildbots.
if os.environ.get('BUILDBOT_BUILDERNAME'):
RunGitCmd(directory, ['config', 'user.name', 'Naclports'])
RunGitCmd(directory,
['config', 'user.email', '*****@*****.**'])
RunGitCmd(directory, ['add', '-f', '.'])
RunGitCmd(directory, ['commit', '-m', 'Upstream version'])
RunGitCmd(directory, ['checkout', '-b', 'upstream'])
RunGitCmd(directory, ['checkout', 'master'])
except: # pylint: disable=bare-except
# If git setup fails or is interrupted then remove the partially
# initialized repository.
util.RemoveTree(os.path.join(git_dir))
def GitCloneToMirror(self):
"""Clone the upstream git repo into a local mirror. """
git_url, git_commit = self.URL.split('@', 2)
# Clone upstream git repo into local mirror, or update the existing
# mirror.
git_mirror = git_url.split('://', 2)[1]
git_mirror = git_mirror.replace('/', '_')
mirror_dir = os.path.join(paths.CACHE_ROOT, git_mirror)
if os.path.exists(mirror_dir):
if RunGitCmd(mirror_dir, ['rev-parse', git_commit + '^{commit}'],
error_ok=True) != 0:
Log('Updating git mirror: %s' % util.RelPath(mirror_dir))
RunGitCmd(mirror_dir, ['remote', 'update', '--prune'])
else:
Log('Mirroring upstream git repo: %s' % self.URL)
RunGitCmd(paths.CACHE_ROOT,
['clone', '--mirror', git_url, git_mirror])
Log('git mirror up-to-date: %s' % util.RelPath(mirror_dir))
return mirror_dir, git_commit
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('revision',
metavar='REVISION',
help='webports revision to to scan for.')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Output extra information.')
parser.add_argument('-p',
'--parallel',
action='store_true',
help='Download packages in parallel.')
parser.add_argument('-l',
'--cache-listing',
action='store_true',
help='Cached output of gsutil -L (for testing).')
parser.add_argument(
'--skip-md5',
action='store_true',
help='Assume on-disk files are up-to-date (for testing).')
args = parser.parse_args(args)
if args.verbose:
webports.SetVerbose(True)
sdk_version = webports.util.GetSDKVersion()
Log('Scanning packages built for pepper_%s at revsion %s' %
(sdk_version, args.revision))
base_path = '%s/builds/pepper_%s/%s/packages' % (
webports.GS_BUCKET, sdk_version, args.revision)
gs_url = 'gs://' + base_path + '/*'
listing_file = os.path.join(webports.NACLPORTS_ROOT, 'lib', 'listing.txt')
if args.cache_listing and os.path.exists(listing_file):
Log('Using pre-cached gs listing: %s' % listing_file)
with open(listing_file) as f:
listing = f.read()
else:
Log('Searching for packages at: %s' % gs_url)
cmd = FindGsutil() + ['stat', gs_url]
LogVerbose('Running: %s' % str(cmd))
try:
listing = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
raise webports.Error("Command '%s' failed: %s" % (cmd, e))
if args.cache_listing:
with open(listing_file, 'w') as f:
f.write(listing)
all_files = ParseGsUtilOutput(listing)
Log('Found %d packages [%s]' %
(len(all_files), FormatSize(sum(f.size for f in all_files))))
binaries = DownloadFiles(all_files, not args.skip_md5, args.parallel)
index_file = os.path.join(webports.NACLPORTS_ROOT, 'lib', 'prebuilt.txt')
Log('Generating %s' % index_file)
webports.package_index.WriteIndex(index_file, binaries)
Log('Done')
return 0
def Extract(self):
"""Extract the package archive into its build location.
This method assumes the package has already been downloaded.
"""
if self.IsGitUpstream():
self.GitClone()
return
archive = self.DownloadLocation()
if not archive:
self.Log('Skipping extract; No upstream archive')
return
dest = self.GetBuildLocation()
output_path, new_foldername = os.path.split(dest)
util.Makedirs(output_path)
# Check existing stamp file contents
stamp_file = self.GetExtractStamp()
stamp_contents = self.GetExtractStampContent()
if os.path.exists(dest):
if StampContentsMatch(stamp_file, stamp_contents):
Log('Already up-to-date: %s' % util.RelPath(dest))
return
raise Error("Upstream archive or patch has changed.\n" +
"Please remove existing checkout and try again: '%s'" %
dest)
util.LogHeading('Extracting')
util.Makedirs(paths.OUT_DIR)
tmp_output_path = tempfile.mkdtemp(dir=paths.OUT_DIR)
try:
ExtractArchive(archive, tmp_output_path)
src = os.path.join(tmp_output_path, new_foldername)
if not os.path.isdir(src):
raise Error('Archive contents not found: %s' % src)
LogVerbose("renaming '%s' -> '%s'" % (src, dest))
os.rename(src, dest)
finally:
util.RemoveTree(tmp_output_path)
self.RemoveStamps()
WriteStamp(stamp_file, stamp_contents)
def Download(self, force_mirror=None):
"""Download upstream sources and verify integrity."""
if self.IsGitUpstream():
self.GitCloneToMirror()
return
archive = self.DownloadLocation()
if not archive:
return
if force_mirror is None:
force_mirror = os.environ.get('FORCE_MIRROR', False)
self.DownloadArchive(force_mirror=force_mirror)
if self.SHA1 is None:
raise PkgFormatError('missing SHA1 attribute: %s' % self.info)
util.VerifyHash(archive, self.SHA1)
Log('verified: %s' % util.RelPath(archive))
def UpdatePatch(self):
if self.URL is None:
return
git_dir = self.GetBuildLocation()
if not os.path.exists(git_dir):
raise Error('Source directory not found: %s' % git_dir)
try:
diff = subprocess.check_output(
['git', 'diff', 'upstream', '--no-ext-diff'], cwd=git_dir)
except subprocess.CalledProcessError as e:
raise Error('error running git in %s: %s' % (git_dir, str(e)))
# Drop index lines for a more stable diff.
diff = re.sub('\nindex [^\n]+\n', '\n', diff)
# Drop binary files, as they don't work anyhow.
diff = re.sub(
'diff [^\n]+\n'
'(new file [^\n]+\n)?'
'(deleted file mode [^\n]+\n)?'
'Binary files [^\n]+ differ\n', '', diff)
# Always filter out config.sub changes
diff_skip = ['*config.sub']
# Add optional per-port skip list.
diff_skip_file = os.path.join(self.root, 'diff_skip.txt')
if os.path.exists(diff_skip_file):
with open(diff_skip_file) as f:
diff_skip += f.read().splitlines()
new_diff = ''
skipping = False
for line in diff.splitlines():
if line.startswith('diff --git a/'):
filename = line[len('diff --git a/'):].split()[0]
skipping = False
for skip in diff_skip:
if fnmatch.fnmatch(filename, skip):
skipping = True
break
if not skipping:
new_diff += line + '\n'
diff = new_diff
# Write back out the diff.
patch_path = self.GetPatchFile()
preexisting = os.path.exists(patch_path)
if not diff:
if preexisting:
Log('removing patch file: %s' % util.RelPath(patch_path))
os.remove(patch_path)
else:
Log('no patch required: %s' % util.RelPath(git_dir))
return
if preexisting:
with open(patch_path) as f:
if diff == f.read():
Log('patch unchanged: %s' % util.RelPath(patch_path))
return
with open(patch_path, 'w') as f:
f.write(diff)
if preexisting:
Log('created patch: %s' % util.RelPath(patch_path))
else:
Log('updated patch: %s' % util.RelPath(patch_path))
def Log(self, message):
Log('%s: %s' % (message, self.InfoString()))