def load_index_file(self, up, url, load):
"""
Fetches layer information from a local file or directory.
The return value is a LayerIndexObj.
ud is the parsed url to the local file or directory.
"""
if not os.path.exists(up.path):
raise FileNotFoundError(up.path)
index = layerindexlib.LayerIndexObj()
index.config = {}
index.config['TYPE'] = self.type
index.config['URL'] = url
params = self.layerindex._parse_params(up.params)
if 'desc' in params:
index.config['DESCRIPTION'] = unquote(params['desc'])
else:
index.config['DESCRIPTION'] = up.path
if 'cache' in params:
index.config['CACHE'] = params['cache']
if 'branch' in params:
branches = params['branch'].split(',')
index.config['BRANCH'] = branches
else:
branches = ['*']
def load_cache(path, index, branches=[]):
logger.debug('Loading json file %s' % path)
with open(path, 'rt', encoding='utf-8') as f:
pindex = json.load(f)
# Filter the branches on loaded files...
newpBranch = []
for branch in branches:
if branch != '*':
if 'branches' in pindex:
for br in pindex['branches']:
if br['name'] == branch:
newpBranch.append(br)
else:
if 'branches' in pindex:
for br in pindex['branches']:
newpBranch.append(br)
if newpBranch:
index.add_raw_element('branches', layerindexlib.Branch,
newpBranch)
else:
logger.debug('No matching branches (%s) in index file(s)' %
branches)
# No matching branches.. return nothing...
return
for (lName, lType) in [
("layerItems", layerindexlib.LayerItem),
("layerBranches", layerindexlib.LayerBranch),
("layerDependencies", layerindexlib.LayerDependency),
("recipes", layerindexlib.Recipe),
("machines", layerindexlib.Machine),
("distros", layerindexlib.Distro)
]:
if lName in pindex:
index.add_raw_element(lName, lType, pindex[lName])
if not os.path.isdir(up.path):
load_cache(up.path, index, branches)
return index
logger.debug('Loading from dir %s...' % (up.path))
for (dirpath, _, filenames) in os.walk(up.path):
for filename in filenames:
if not filename.endswith('.json'):
continue
fpath = os.path.join(dirpath, filename)
load_cache(fpath, index, branches)
return index
def _load_bblayers(self, branches=None):
"""Load the BBLAYERS and related collection information"""
d = self.layerindex.data
if not branches:
raise LayerIndexFetchError(
"No branches specified for _load_bblayers!")
index = layerindexlib.LayerIndexObj()
branchId = 0
index.branches = {}
layerItemId = 0
index.layerItems = {}
layerBranchId = 0
index.layerBranches = {}
bblayers = d.getVar('BBLAYERS').split()
if not bblayers:
# It's blank! Nothing to process...
return index
collections = d.getVar('BBFILE_COLLECTIONS')
layerconfs = d.varhistory.get_variable_items_files(
'BBFILE_COLLECTIONS', d)
bbfile_collections = {
layer: os.path.dirname(os.path.dirname(path))
for layer, path in layerconfs.items()
}
(_, bb_branch, _, _) = self._get_bitbake_info()
for branch in branches:
branchId += 1
index.branches[branchId] = layerindexlib.Branch(index, None)
index.branches[branchId].define_data(branchId, branch, bb_branch)
for entry in collections.split():
layerpath = entry
if entry in bbfile_collections:
layerpath = bbfile_collections[entry]
layername = d.getVar('BBLAYERS_LAYERINDEX_NAME_%s' %
entry) or os.path.basename(layerpath)
layerversion = d.getVar('LAYERVERSION_%s' % entry) or ""
layerurl = self._handle_git_remote(layerpath)
layersubdir = ""
layerrev = "<unknown>"
layerbranch = "<unknown>"
if os.path.isdir(layerpath):
layerbasepath = self._run_command(
'git rev-parse --show-toplevel',
layerpath,
default=layerpath)
if os.path.abspath(layerpath) != os.path.abspath(
layerbasepath):
layersubdir = os.path.abspath(
layerpath)[len(layerbasepath) + 1:]
layerbranch = self._run_command(
'git rev-parse --abbrev-ref HEAD',
layerpath,
default="<unknown>")
layerrev = self._run_command('git rev-parse HEAD',
layerpath,
default="<unknown>")
for remotes in self._run_command('git remote -v',
layerpath,
default="").split("\n"):
if not remotes:
layerurl = self._handle_git_remote(layerpath)
else:
remote = remotes.split("\t")[1].split(" ")[0]
if "(fetch)" == remotes.split("\t")[1].split(" ")[1]:
layerurl = self._handle_git_remote(remote)
break
layerItemId += 1
index.layerItems[layerItemId] = layerindexlib.LayerItem(
index, None)
index.layerItems[layerItemId].define_data(layerItemId,
layername,
description=layerpath,
vcs_url=layerurl)
for branchId in index.branches:
layerBranchId += 1
index.layerBranches[layerBranchId] = layerindexlib.LayerBranch(
index, None)
index.layerBranches[layerBranchId].define_data(
layerBranchId,
entry,
layerversion,
layerItemId,
branchId,
vcs_subdir=layersubdir,
vcs_last_rev=layerrev,
actual_branch=layerbranch)
return index
def load_index_web(self, up, url, load):
"""
Fetches layer information from a remote layer index.
The return value is a LayerIndexObj.
ud is the parsed url to the rest api of the layer index, such as:
http://layers.openembedded.org/layerindex/api/
"""
def _get_json_response(apiurl=None,
username=None,
password=None,
retry=True):
assert apiurl is not None
logger.debug("fetching %s" % apiurl)
up = urlparse(apiurl)
username = up.username
password = up.password
# Strip username/password and params
if up.port:
up_stripped = up._replace(params="",
netloc="%s:%s" %
(up.hostname, up.port))
else:
up_stripped = up._replace(params="", netloc=up.hostname)
res = self.layerindex._fetch_url(up_stripped.geturl(),
username=username,
password=password)
try:
parsed = json.loads(res.read().decode('utf-8'))
except ConnectionResetError:
if retry:
logger.debug("%s: Connection reset by peer. Retrying..." %
url)
parsed = _get_json_response(apiurl=up_stripped.geturl(),
username=username,
password=password,
retry=False)
logger.debug("%s: retry successful.")
else:
raise layerindexlib.LayerIndexFetchError(
'%s: Connection reset by peer. Is there a firewall blocking your connection?'
% apiurl)
return parsed
index = layerindexlib.LayerIndexObj()
index.config = {}
index.config['TYPE'] = self.type
index.config['URL'] = url
params = self.layerindex._parse_params(up.params)
if 'desc' in params:
index.config['DESCRIPTION'] = unquote(params['desc'])
else:
index.config['DESCRIPTION'] = up.hostname
if 'cache' in params:
index.config['CACHE'] = params['cache']
if 'branch' in params:
branches = params['branch'].split(',')
index.config['BRANCH'] = branches
else:
branches = ['*']
try:
index.apilinks = _get_json_response(apiurl=url,
username=up.username,
password=up.password)
except Exception as e:
raise layerindexlib.LayerIndexFetchError(url, e)
# Local raw index set...
pindex = {}
# Load all the requested branches at the same time time,
# a special branch of '*' means load all branches
filter = ""
if "*" not in branches:
filter = "?filter=name:%s" % "OR".join(branches)
logger.debug("Loading %s from %s" %
(branches, index.apilinks['branches']))
# The link won't include username/password, so pull it from the original url
pindex['branches'] = _get_json_response(index.apilinks['branches'] +
filter,
username=up.username,
password=up.password)
if not pindex['branches']:
logger.debug("No valid branches (%s) found at url %s." %
(branch, url))
return index
index.add_raw_element("branches", layerindexlib.Branch,
pindex['branches'])
# Load all of the layerItems (these can not be easily filtered)
logger.debug("Loading %s from %s" %
('layerItems', index.apilinks['layerItems']))
# The link won't include username/password, so pull it from the original url
pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'],
username=up.username,
password=up.password)
if not pindex['layerItems']:
logger.debug("No layers were found at url %s." % (url))
return index
index.add_raw_element("layerItems", layerindexlib.LayerItem,
pindex['layerItems'])
# From this point on load the contents for each branch. Otherwise we
# could run into a timeout.
for branch in index.branches:
filter = "?filter=branch__name:%s" % index.branches[branch].name
logger.debug("Loading %s from %s" %
('layerBranches', index.apilinks['layerBranches']))
# The link won't include username/password, so pull it from the original url
pindex['layerBranches'] = _get_json_response(
index.apilinks['layerBranches'] + filter,
username=up.username,
password=up.password)
if not pindex['layerBranches']:
logger.debug("No valid layer branches (%s) found at url %s." %
(branches or "*", url))
return index
index.add_raw_element("layerBranches", layerindexlib.LayerBranch,
pindex['layerBranches'])
# Load the rest, they all have a similar format
# Note: the layer index has a few more items, we can add them if necessary
# in the future.
filter = "?filter=layerbranch__branch__name:%s" % index.branches[
branch].name
for (lName, lType) in [("layerDependencies",
layerindexlib.LayerDependency),
("recipes", layerindexlib.Recipe),
("machines", layerindexlib.Machine),
("distros", layerindexlib.Distro)]:
if lName not in load:
continue
logger.debug("Loading %s from %s" %
(lName, index.apilinks[lName]))
# The link won't include username/password, so pull it from the original url
pindex[lName] = _get_json_response(index.apilinks[lName] +
filter,
username=up.username,
password=up.password)
index.add_raw_element(lName, lType, pindex[lName])
return index
