def git_branch(_dst, _orig_branch, _branch):
logger.info('Branching %s: %s -> %s' % (_dst, _orig_branch, _branch))
_cmd = ['git', 'fetch', '.', '%s:%s' % (_orig_branch, _branch)]
if force:
_cmd.append('-f')
utils_setup.run_cmd(_cmd, cwd=_dst)
work_list.append('%s %s' % (_dst, _branch))
def commit_files(self):
logger.debug('Starting')
# List of all files that may change due to config
filelist = [
'README',
'default.xml',
'.gitignore',
'.gitconfig',
]
# If we are mirroring, skip all of these...
if self.mirror != True:
filelist.append('layers/local')
filelist.append('.templateconf')
filelist.append('config/bblayers.conf.sample')
filelist.append('config/conf-notes.txt')
filelist.append('config/local.conf.sample')
if os.path.exists('config/site.conf.sample'):
filelist.append('config/site.conf.sample')
# Add log dir if it contains files
if os.listdir('config/log'):
filelist.append('config/log')
# git init
if not os.path.exists(self.project_dir + '/.git'):
cmd = [self.tools['git'], 'init', self.project_dir]
if self.quiet == self.default_repo_quiet:
cmd.append(self.quiet)
utils_setup.run_cmd(cmd, environment=self.env, cwd=self.conf_dir)
# Add self.install_dir as a submodule if it is in self.project_dir
if self.install_dir.startswith(self.project_dir + '/'):
logger.debug('Add %s as a submodule' % self.install_dir)
cmd = [self.tools['git'], 'submodule', 'add', \
'./' + os.path.relpath(self.install_dir, self.project_dir)]
utils_setup.run_cmd(cmd, environment=self.env, cwd=self.project_dir)
filelist.append(self.install_dir)
filelist.append('.gitmodules')
# git add manifest. (Since these files are new, always try to add them)
cmd = [self.tools['git'], 'add', '--'] + filelist
utils_setup.run_cmd(cmd, environment=self.env, cwd=self.project_dir)
try:
cmd = [self.tools['git'], 'diff-index', '--quiet', 'HEAD', '--'] + filelist
utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=self.project_dir, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
except:
logger.plain('Updated project configuration')
# Command failed -- so self.default_xml changed...
cmd = [self.tools['git'], 'commit', '-m', 'Configuration change - %s' % (self.setup_args), '--'] + filelist
utils_setup.run_cmd(cmd, environment=self.env, cwd=self.project_dir)
logger.debug('Done')
def update_mirror(_dst_mirror):
cmd = ['git', 'add', '-A', '.']
utils_setup.run_cmd(cmd, cwd=_dst_mirror)
logger.debug('Updating mirror-index')
cmd = ['git', 'commit', '-m', 'Updated index - Flatten Mirror']
try:
utils_setup.run_cmd(cmd, cwd=_dst_mirror)
except:
# Nothing changed...
pass
def call_repo_sync(self, args):
logger.debug('Starting')
repo = self.tools['repo']
cmd = args
cmd.insert(0, repo)
cmd.insert(1, 'sync')
if self.force_sync:
cmd.append(self.force_sync)
log_it = 1
if self.repo_verbose is not True and self.quiet == self.default_repo_quiet:
cmd.append(self.quiet)
log_it = 0
utils_setup.run_cmd(cmd, environment=self.env, log=log_it)
logger.debug('Done')
def push_or_copy(_layer, _src, _dst, _branch=None):
if subset_folders:
if _layer not in subset_folders:
logger.critical("Layer %s (%s) not in SUBSET_FOLDERS" % (_layer, _src))
raise
elif subset_folders[_layer] == "[SKIP]":
return
else:
dstdir = os.path.dirname(_dst)
dstbase = os.path.basename(_dst)
_dst = os.path.join(dstdir, subset_folders[_layer], dstbase)
if not os.path.exists(_src):
_src += '.git'
if not os.path.exists(_src):
logger.critical("Unable to find %s!" % _src)
raise
# Make output consistent
if not strip_git and not _dst.endswith('.git'):
_dst += '.git'
if strip_git and _dst.endswith('.git'):
_dst = _dst[:-4]
if not git_push or not _branch:
logger.plain('cp %s -> %s' % (_src, _dst))
shutil.copytree(_src, _dst, symlinks=True, ignore_dangling_symlinks=True)
else:
logger.plain('push %s -> %s (%s)' % (_src, _dst, _branch))
if os.path.exists(_dst):
logger.critical('Destination %s already exists!' % _dst)
raise
os.makedirs(_dst, exist_ok=True)
# New bare repo
_cmd = [ 'git', 'init', '--bare' ]
utils_setup.run_cmd(_cmd, cwd=_dst)
# Push just the one branch
_cmd = [ 'git', 'push', os.path.abspath(_dst), _branch ]
utils_setup.run_cmd(_cmd, cwd=_src)
def call_repo_init(self, args):
logger.debug('Starting')
repo = self.tools['repo']
directory = os.path.join(self.project_dir, self.check_repo_install_dir)
if os.path.exists(directory):
logger.info('Done: detected repo init already run since %s exists' % directory)
return
cmd = args
cmd.insert(0, repo)
cmd.insert(1, 'init')
if self.depth:
cmd.append(self.depth)
log_it = 1
if self.repo_verbose is not True and self.quiet == self.default_repo_quiet:
cmd.append(self.quiet)
log_it = 0
try:
utils_setup.run_cmd(cmd, environment=self.env, log=log_it)
except Exception as e:
raise e
logger.debug('Done')
def get_mirror_dir(_layer, _dst):
if subset_folders:
if _layer in subset_folders:
if subset_folders[_layer] == "[SKIP]":
return None
dstdir = os.path.dirname(_dst)
dstbase = os.path.basename(_dst)
_dst = os.path.join(dstdir, subset_folders[_layer], dstbase)
else:
logger.error("Layer %s not in SUBSET_FOLDERS" % _layer)
# Use default directory
# If the target directory does not exist, create it
if not os.path.exists(_dst):
cmd = ['git', 'init', _dst]
utils_setup.run_cmd(cmd)
if not branch or branch == "":
logger.critical('Where did the branch val go!? %s' % branch)
raise
cmd = ['git', 'checkout', '-b', branch]
try:
utils_setup.run_cmd(cmd, cwd=_dst)
except:
# if we failed, then simply try to switch branches
cmd = ['git', 'checkout', branch]
utils_setup.run_cmd(cmd, cwd=_dst)
return _dst
def write_local_mirror_index(self, setup, mirror_index_path):
import subprocess
import utils_setup
# We need access to the sortRestApi function...
from layer_index import Layer_Index
li = Layer_Index()
# We want to move to a generic named branch, now that we've done the fixups.
try:
cmd = [setup.tools['git'], 'checkout', '--orphan', setup.base_branch ]
utils_setup.run_cmd(cmd, environment=setup.env, cwd=mirror_index_path, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
except Exception:
cmd = [setup.tools['git'], 'checkout', setup.base_branch ]
utils_setup.run_cmd(cmd, log=2, environment=setup.env, cwd=mirror_index_path)
cmd = [setup.tools['git'], 'reset', '--hard' ]
utils_setup.run_cmd(cmd, log=2, environment=setup.env, cwd=mirror_index_path)
# Remove obsolete entries only
for (dirpath, _, filenames) in os.walk(mirror_index_path):
if dirpath.endswith('/.git') or '/.git/' in dirpath:
continue
for filename in filenames:
if filename not in self.indexes and filename not in self.xmls:
logger.debug('ws mirror-index remove obsolete %s' % os.path.join(dirpath, filename))
os.remove(os.path.join(dirpath, filename))
for entry in self.indexes:
logger.debug('Writing windshare index %s...' % entry)
fpath = os.path.join(mirror_index_path, entry)
json.dump(li.sortRestApi(self.indexes[entry]), open(fpath, 'wt'), indent=4)
for entry in self.xmls:
logger.debug('Writing windshare xml %s...' % entry)
os.makedirs(os.path.join(mirror_index_path, 'xml'), exist_ok=True)
fpath = os.path.join(mirror_index_path, 'xml', entry)
with open(fpath, 'wt') as fout:
for _line in self.xmls[entry]:
fout.write(_line + '\n')
cmd = [setup.tools['git'], 'add', '-A', '.']
utils_setup.run_cmd(cmd, log=2, environment=setup.env, cwd=mirror_index_path)
try:
cmd = [setup.tools['git'], 'diff-index', '--quiet', 'HEAD', '--']
utils_setup.run_cmd(cmd, environment=setup.env, cwd=mirror_index_path, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
except Exception:
# We expect to fail to this code
logger.debug('Updating windshare mirror-index')
cmd = [setup.tools['git'], 'commit', '-m', 'Updated index - %s' % (setup.setup_args)]
utils_setup.run_cmd(cmd, log=2, environment=setup.env, cwd=mirror_index_path)
def update_mirror_index(self):
logger.debug('Starting')
path = os.path.join(self.project_dir, 'mirror-index')
logger.plain('Exporting mirror-index %s...' % (path))
if not os.path.exists(path):
cmd = [self.tools['git'], 'init', path]
if self.quiet == self.default_repo_quiet:
cmd.append(self.quiet)
utils_setup.run_cmd(cmd,
environment=self.env,
cwd=self.project_dir)
cmd = [self.tools['git'], 'checkout', '-b', self.base_branch]
ret = subprocess.Popen(cmd,
env=self.env,
cwd=path,
close_fds=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
ret.wait()
if (ret.returncode != 0):
# if we failed, then simply try to switch branches
cmd = [self.tools['git'], 'checkout', self.base_branch]
ret = subprocess.Popen(cmd,
env=self.env,
cwd=path,
close_fds=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
ret.wait()
# Make sure the directory is empty, use -f to ignore failures
for (dirpath, dirnames, filenames) in os.walk(path):
if dirpath.endswith('/.git') or path + '/.git' in dirpath:
continue
for filename in filenames:
os.remove(os.path.join(dirpath, filename))
# Construct a list of all layers we've downloaded, by url, including sublayers not activated
url_cache = {}
for (lindex,
layerBranch) in self.requiredlayers + self.recommendedlayers:
for layer in self.index.find_layer(lindex,
id=layerBranch['layer']):
vcs_url = layer['vcs_url']
if not vcs_url in url_cache:
url_cache[vcs_url] = []
url_cache[vcs_url].append((lindex, layerBranch['branch']))
# Serialize the information for each of the layers (and their sublayers)
for vcs_url in url_cache:
for (lindex, branchid) in url_cache[vcs_url]:
for layer in lindex['layerItems']:
if layer['vcs_url'] in url_cache:
for lb in self.index.getLayerBranch(lindex,
branchid=branchid,
layerItem=layer):
self.index.serialize_index(
lindex,
os.path.join(path,
lindex['CFG']['DESCRIPTION']),
split=True,
layerBranches=[lb],
IncludeCFG=True,
mirror=True,
base_url=self.base_url)
name = layer['name']
destdir = os.path.join(path, 'xml')
srcfile = os.path.join(self.xml_dir, '%s.inc' % (name))
if os.path.exists(srcfile):
os.makedirs(destdir, exist_ok=True)
shutil.copy(srcfile, destdir)
srcfile = os.path.join(self.xml_dir, '%s.xml' % (name))
if os.path.exists(srcfile):
os.makedirs(destdir, exist_ok=True)
shutil.copy(srcfile, destdir)
# Special processing for the openembedded-core layer
if name == 'openembedded-core':
srcfile = os.path.join(self.xml_dir, 'bitbake.inc')
if os.path.exists(srcfile):
os.makedirs(destdir, exist_ok=True)
shutil.copy(srcfile, destdir)
srcfile = os.path.join(self.xml_dir, 'bitbake.xml')
if os.path.exists(srcfile):
os.makedirs(destdir, exist_ok=True)
shutil.copy(srcfile, destdir)
# git add file.
cmd = [self.tools['git'], 'add', '-A', '.']
utils_setup.run_cmd(cmd, environment=self.env, cwd=path)
cmd = [self.tools['git'], 'diff-index', '--quiet', 'HEAD', '--']
ret = subprocess.Popen(cmd,
env=self.env,
cwd=path,
close_fds=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
ret.wait()
if (ret.returncode != 0):
logger.debug('Updating mirror-index')
cmd = [
self.tools['git'], 'commit', '-m',
'Updated index - %s' % (self.setup_args)
]
utils_setup.run_cmd(cmd, environment=self.env, cwd=path)
logger.debug('Done')
def load_layer_index(self):
# Load Layer_Index
replace = []
replace = replace + settings.REPLACE
replace = replace + [
('#INSTALL_DIR#', self.install_dir),
('#BASE_URL#', self.base_url),
('#BASE_BRANCH#', self.base_branch),
]
# See if there is a mirror index available from the BASE_URL
mirror_index_path = None
mirror_index = os.path.join(self.conf_dir, 'mirror-index')
cmd = [
self.tools['git'], 'ls-remote', self.base_url + '/mirror-index',
self.base_branch
]
ret = subprocess.Popen(cmd,
env=self.env,
cwd=self.project_dir,
close_fds=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
ret.wait()
if (ret.returncode == 0):
logger.plain('Loading the mirror index from %s (%s)...' %
(self.base_url + '/mirror-index', self.base_branch))
# This MIGHT be a valid mirror..
if not os.path.exists(mirror_index):
os.makedirs(mirror_index)
cmd = [self.tools['git'], 'init']
utils_setup.run_cmd(cmd,
environment=self.env,
cwd=mirror_index)
cmd = [
self.tools['git'], 'fetch', '-f', '-n', '-u',
self.base_url + '/mirror-index',
self.base_branch + ':' + self.base_branch
]
ret = subprocess.Popen(cmd,
env=self.env,
cwd=mirror_index,
close_fds=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
ret.wait()
if (ret.returncode == 0):
logger.debug('Found mirrored index.')
cmd = [self.tools['git'], 'checkout', self.base_branch]
utils_setup.run_cmd(cmd,
environment=self.env,
cwd=mirror_index)
cmd = [self.tools['git'], 'reset', '--hard']
utils_setup.run_cmd(cmd,
environment=self.env,
cwd=mirror_index)
mirror_index_path = mirror_index
# Mirror also has a copy of the associated XML bits
self.xml_dir = os.path.join(mirror_index, 'xml')
self.index = Layer_Index(indexcfg=settings.INDEXES,
base_branch=self.base_branch,
replace=replace,
mirror=mirror_index_path)
def load_mirror_index(self, remote_mirror):
# See if there is a mirror index available from the BASE_URL
mirror_index = os.path.join(self.conf_dir, 'mirror-index')
try:
cmd = [self.tools['git'], 'ls-remote', remote_mirror, self.base_branch]
utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=self.project_dir, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
except:
try:
remote_mirror += "/.git"
cmd = [self.tools['git'], 'ls-remote', remote_mirror, self.base_branch]
utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=self.project_dir, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
except:
# No mirror, return
return None
logger.plain('Loading the mirror index from %s (%s)...' % (remote_mirror, self.base_branch))
# This MIGHT be a valid mirror..
if not os.path.exists(mirror_index):
os.makedirs(mirror_index)
cmd = [self.tools['git'], 'init' ]
utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=mirror_index)
try:
cmd = [self.tools['git'], 'fetch', '-f', '-n', '-u', remote_mirror, self.base_branch + ':' + self.base_branch]
utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=mirror_index)
except:
# Could not fetch, return
return None
logger.debug('Found mirrored index.')
cmd = [self.tools['git'], 'checkout', self.base_branch ]
utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=mirror_index)
cmd = [self.tools['git'], 'reset', '--hard' ]
utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=mirror_index)
return mirror_index
# Branch the setup program....
git_branch(_dst=dst, _orig_branch=branch, _branch=dest_branch)
completed.append(dst)
# Transform and export the mirror index
git_branch(mirror_path, branch, dest_branch)
completed.append(mirror_path)
index = Layer_Index(indexcfg=settings.INDEXES,
base_branch=branch,
replace=settings.REPLACE,
mirror=mirror_path)
cmd = ['git', 'checkout', dest_branch]
utils_setup.run_cmd(cmd, cwd=mirror_path)
logger.info('Loading default.xml')
tree = ET.parse('default.xml')
root = tree.getroot()
logger.info('Branching based on default.xml')
default_revision = None
base_url = None
for child in root:
if child.tag == 'remote':
if 'fetch' in child.attrib:
base_url = child.attrib['fetch']
if child.tag == 'default':
if 'revision' in child.attrib:
