def git_clone_to_mirror(self):
"""Clone the upstream git repo into a local mirror. """
git_url, git_commit = self.URL.split('@', 2)
# Clone upstream git repo into local mirror, or update the existing
# mirror.
git_mirror = git_url.split('://', 2)[1]
git_mirror = git_mirror.replace('/', '_')
mirror_dir = os.path.join(paths.CACHE_ROOT, git_mirror)
if os.path.exists(mirror_dir):
if run_git_cmd(mirror_dir, ['rev-parse', git_commit + '^{commit}'],
error_ok=True) != 0:
log('Updating git mirror: %s' % util.rel_path(mirror_dir))
run_git_cmd(mirror_dir, ['remote', 'update', '--prune'])
else:
log('Mirroring upstream git repo: %s' % self.URL)
run_git_cmd(paths.CACHE_ROOT, ['clone', '--mirror', git_url, git_mirror])
log('git mirror up-to-date: %s' % util.rel_path(mirror_dir))
return mirror_dir, git_commit
def extract(self):
"""Extract the package archive into its build location.
This method assumes the package has already been downloaded.
"""
if self.is_git_upstream():
self.git_clone()
return
archive = self.download_location()
if not archive:
self.log('Skipping extract; No upstream archive')
return
dest = self.get_build_location()
output_path, new_foldername = os.path.split(dest)
util.makedirs(output_path)
# Check existing stamp file contents
stamp_file = self.get_extract_stamp()
stamp_contents = self.get_extract_stamp_content()
if os.path.exists(dest):
if stamp_contents_match(stamp_file, stamp_contents):
log('Already up-to-date: %s' % util.rel_path(dest))
return
raise Error("Upstream archive or patch has changed.\n" +
"Please remove existing checkout and try again: '%s'" %
dest)
util.log_heading('Extracting')
util.makedirs(paths.OUT_DIR)
tmp_output_path = tempfile.mkdtemp(dir=paths.OUT_DIR)
try:
extract_archive(archive, tmp_output_path)
src = os.path.join(tmp_output_path, new_foldername)
if not os.path.isdir(src):
raise Error('Archive contents not found: %s' % src)
log_verbose("renaming '%s' -> '%s'" % (src, dest))
os.rename(src, dest)
finally:
util.remove_tree(tmp_output_path)
self.remove_stamps()
write_stamp(stamp_file, stamp_contents)
def extract(self):
"""Extract the package archive into its build location.
This method assumes the package has already been downloaded.
"""
if self.is_git_upstream():
self.git_clone()
return
archive = self.download_location()
if not archive:
self.log('Skipping extract; No upstream archive')
return
dest = self.get_build_location()
output_path, new_foldername = os.path.split(dest)
util.makedirs(output_path)
# Check existing stamp file contents
stamp_file = self.get_extract_stamp()
stamp_contents = self.get_extract_stamp_content()
if os.path.exists(dest):
if stamp_contents_match(stamp_file, stamp_contents):
log('Already up-to-date: %s' % util.rel_path(dest))
return
raise Error("Upstream archive or patch has changed.\n" +
"Please remove existing checkout and try again: '%s'" % dest)
util.log_heading('Extracting')
util.makedirs(paths.OUT_DIR)
tmp_output_path = tempfile.mkdtemp(dir=paths.OUT_DIR)
try:
extract_archive(archive, tmp_output_path)
src = os.path.join(tmp_output_path, new_foldername)
if not os.path.isdir(src):
raise Error('Archive contents not found: %s' % src)
log_verbose("renaming '%s' -> '%s'" % (src, dest))
os.rename(src, dest)
finally:
util.remove_tree(tmp_output_path)
self.remove_stamps()
write_stamp(stamp_file, stamp_contents)
def download(self, force_mirror=None):
"""Download upstream sources and verify integrity."""
if self.is_git_upstream():
self.git_clone_to_mirror()
return
archive = self.download_location()
if not archive:
return
if force_mirror is None:
force_mirror = os.environ.get('FORCE_MIRROR', False)
self.download_archive(force_mirror=force_mirror)
if self.SHA1 is None:
raise PkgFormatError('missing SHA1 attribute: %s' % self.info)
util.verify_hash(archive, self.SHA1)
log('verified: %s' % util.rel_path(archive))
def download(self, force_mirror=None):
"""Download upstream sources and verify integrity."""
if self.is_git_upstream():
self.git_clone_to_mirror()
return
archive = self.download_location()
if not archive:
return
if force_mirror is None:
force_mirror = os.environ.get('FORCE_MIRROR', False)
self.download_archive(force_mirror=force_mirror)
if self.SHA1 is None:
raise PkgFormatError('missing SHA1 attribute: %s' % self.info)
util.verify_hash(archive, self.SHA1)
log('verified: %s' % util.rel_path(archive))
def update_patch(self):
if self.URL is None:
return
git_dir = self.get_build_location()
if not os.path.exists(git_dir):
raise Error('Source directory not found: %s' % git_dir)
try:
diff = subprocess.check_output(
['git', 'diff', 'upstream', '--no-ext-diff'], cwd=git_dir)
except subprocess.CalledProcessError as e:
raise Error('error running git in %s: %s' % (git_dir, str(e)))
# Drop index lines for a more stable diff.
diff = re.sub('\nindex [^\n]+\n', '\n', diff)
# Drop binary files, as they don't work anyhow.
diff = re.sub(
'diff [^\n]+\n'
'(new file [^\n]+\n)?'
'(deleted file mode [^\n]+\n)?'
'Binary files [^\n]+ differ\n', '', diff)
# Always filter out config.sub changes
diff_skip = ['*config.sub']
# Add optional per-port skip list.
diff_skip_file = os.path.join(self.root, 'diff_skip.txt')
if os.path.exists(diff_skip_file):
with open(diff_skip_file) as f:
diff_skip += f.read().splitlines()
new_diff = ''
skipping = False
for line in diff.splitlines():
if line.startswith('diff --git a/'):
filename = line[len('diff --git a/'):].split()[0]
skipping = False
for skip in diff_skip:
if fnmatch.fnmatch(filename, skip):
skipping = True
break
if not skipping:
new_diff += line + '\n'
diff = new_diff
# Write back out the diff.
patch_path = self.get_patch_file()
preexisting = os.path.exists(patch_path)
if not diff:
if preexisting:
log('removing patch file: %s' % util.rel_path(patch_path))
os.remove(patch_path)
else:
log('no patch required: %s' % util.rel_path(git_dir))
return
if preexisting:
with open(patch_path) as f:
if diff == f.read():
log('patch unchanged: %s' % util.rel_path(patch_path))
return
with open(patch_path, 'w') as f:
f.write(diff)
if preexisting:
log('created patch: %s' % util.rel_path(patch_path))
else:
log('updated patch: %s' % util.rel_path(patch_path))
def test_rel_path(self):
self.assertEqual('bar', util.rel_path('/foo/bar'))
self.assertEqual('../baz/bar', util.rel_path('/baz/bar'))
def update_patch(self):
if self.URL is None:
return
git_dir = self.get_build_location()
if not os.path.exists(git_dir):
raise Error('Source directory not found: %s' % git_dir)
try:
diff = subprocess.check_output(
['git', 'diff', 'upstream', '--no-ext-diff'], cwd=git_dir)
except subprocess.CalledProcessError as e:
raise Error('error running git in %s: %s' % (git_dir, str(e)))
# Drop index lines for a more stable diff.
diff = re.sub('\nindex [^\n]+\n', '\n', diff)
# Drop binary files, as they don't work anyhow.
diff = re.sub('diff [^\n]+\n'
'(new file [^\n]+\n)?'
'(deleted file mode [^\n]+\n)?'
'Binary files [^\n]+ differ\n', '', diff)
# Always filter out config.sub changes
diff_skip = ['*config.sub']
# Add optional per-port skip list.
diff_skip_file = os.path.join(self.root, 'diff_skip.txt')
if os.path.exists(diff_skip_file):
with open(diff_skip_file) as f:
diff_skip += f.read().splitlines()
new_diff = ''
skipping = False
for line in diff.splitlines():
if line.startswith('diff --git a/'):
filename = line[len('diff --git a/'):].split()[0]
skipping = False
for skip in diff_skip:
if fnmatch.fnmatch(filename, skip):
skipping = True
break
if not skipping:
new_diff += line + '\n'
diff = new_diff
# Write back out the diff.
patch_path = self.get_patch_file()
preexisting = os.path.exists(patch_path)
if not diff:
if preexisting:
log('removing patch file: %s' % util.rel_path(patch_path))
os.remove(patch_path)
else:
log('no patch required: %s' % util.rel_path(git_dir))
return
if preexisting:
with open(patch_path) as f:
if diff == f.read():
log('patch unchanged: %s' % util.rel_path(patch_path))
return
with open(patch_path, 'w') as f:
f.write(diff)
if preexisting:
log('created patch: %s' % util.rel_path(patch_path))
else:
log('updated patch: %s' % util.rel_path(patch_path))
