def _get_recipes_filenames(ct, repodir, layerdir, logger):
import glob
ct_files = []
layerdir_start = os.path.normpath(layerdir) + os.sep
files = utils.runcmd(
['git', 'log', '--name-only', '--format=%n', '-n', '1', ct],
repodir,
logger=logger)
incdirs = []
for f in files.split("\n"):
if f != "":
fullpath = os.path.join(repodir, f)
# Skip deleted files in commit
if not os.path.exists(fullpath):
continue
if not fullpath.startswith(layerdir_start):
# Ignore files in repo that are outside of the layer
continue
(typename, _,
filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
if typename == 'recipe':
ct_files.append(fullpath)
elif fullpath.endswith('.inc'):
fpath = os.path.dirname(fullpath)
if not fpath in incdirs:
incdirs.append(fpath)
for fpath in incdirs:
# Let's just assume that all .bb files next to a .inc need to be checked
for f in glob.glob(os.path.join(fpath, '*.bb')):
if not f in ct_files:
ct_files.append(f)
return ct_files
def checkout_layer_deps(layerbranch, commit, fetchdir, logger):
""" Check out the repositories for a layer and its dependencies """
# Some layers will be in the same repository, so we only want to check those out once
done_repos = []
def checkout_layer(lb, lcommit=None, lcommitdate=None, force=False):
urldir = str(lb.layer.get_fetch_dir())
repodir = os.path.join(fetchdir, urldir)
if not repodir in done_repos:
if not lcommit:
lcommit = utils.runcmd([
'git', 'rev-list', '-1',
'--before=%s' % lcommitdate, 'origin/master'
],
repodir,
logger=logger).strip()
utils.checkout_repo(repodir, lcommit, logger, force)
done_repos.append(repodir)
# We "force" here because it's almost certain we'll be checking out a
# different revision for the layer itself
checkout_layer(layerbranch, commit, force=True)
layer_urldir = str(layerbranch.layer.get_fetch_dir())
layer_repodir = os.path.join(fetchdir, layer_urldir)
commitdate = utils.runcmd(['git', 'show', '-s', '--format=%ci'],
layer_repodir,
logger=logger)
for dep in layerbranch.get_recursive_dependencies():
checkout_layer(dep, lcommitdate=commitdate)
return commitdate
def run_internal(maintplanlayerbranch,
commit,
commitdate,
options,
logger,
bitbake_map,
initial=False):
from layerindex.models import PythonEnvironment
from rrs.models import Release
if commitdate < maintplanlayerbranch.python3_switch_date:
# Python 2
if maintplanlayerbranch.python2_environment:
cmdprefix = maintplanlayerbranch.python2_environment.get_command()
else:
cmdprefix = 'python'
# Ensure we're using a bitbake version that is python 2 compatible
if commitdate > datetime(2016, 5, 10):
commitdate = datetime(2016, 5, 10)
else:
# Python 3
if maintplanlayerbranch.python3_environment:
cmdprefix = maintplanlayerbranch.python3_environment.get_command()
else:
cmdprefix = 'python3'
bitbake_rev = utils.runcmd([
'git', 'rev-list', '-1',
'--before=%s' % str(commitdate), 'origin/master'
],
bitbakepath,
logger=logger)
check_rev = bitbake_map.get(bitbake_rev, None)
if check_rev:
logger.debug('Preferring bitbake revision %s over %s' %
(check_rev, bitbake_rev))
bitbake_rev = check_rev
cmd = '%s upgrade_history_internal.py %s %s' % (
cmdprefix, maintplanlayerbranch.layerbranch.id, commit)
if initial:
release = Release.get_by_date(maintplanlayerbranch.plan, commitdate)
if release:
comment = 'Initial import at %s release start.' % release.name
else:
comment = 'Initial import at %s' % commit
cmd += ' --initial="%s"' % comment
if bitbake_rev:
cmd += ' --bitbake-rev %s' % bitbake_rev
if options.filter_files:
cmd += ' --filter-files %s' % options.filter_files
if options.dry_run:
cmd += ' --dry-run'
if options.loglevel == logging.DEBUG:
cmd += ' --debug'
logger.debug('Running %s' % cmd)
ret, output = utils.run_command_interruptible(cmd)
if ret == 254:
# Interrupted by user, break out of loop
logger.info('Update interrupted, exiting')
sys.exit(254)
def remap_range(start, end, replacewith=None):
if replacewith is None:
replacewith = end
bitbake_commits = utils.runcmd(
['git', 'rev-list',
'%s^..%s^' % (start, end)],
bitbakepath,
logger=logger)
bitbake_commit_list = bitbake_commits.splitlines()
for commit in bitbake_commit_list:
bitbake_map[commit] = replacewith
def checkout_layer(lb, lcommit=None, lcommitdate=None, force=False):
urldir = str(lb.layer.get_fetch_dir())
repodir = os.path.join(fetchdir, urldir)
if not repodir in done_repos:
if not lcommit:
lcommit = utils.runcmd(
'git rev-list -1 --before="%s" origin/master' %
lcommitdate,
repodir,
logger=logger).strip()
utils.checkout_repo(repodir, lcommit, logger, force)
done_repos.append(repodir)
def checkout_layer_deps(layerbranch, commit, fetchdir, logger):
""" Check out the repositories for a layer and its dependencies """
oecore_map = {}
if layerbranch.layer.name != 'openembedded-core':
# Filter out some bad commits for OE-Core
for good_rev, bad_revs in oecore_bad_revs.items():
for bad_rev in bad_revs:
oecore_map[bad_rev] = good_rev
# Some layers will be in the same repository, so we only want to check those out once
done_repos = []
def checkout_layer(lb, lcommit=None, lcommitdate=None, force=False):
urldir = str(lb.layer.get_fetch_dir())
repodir = os.path.join(fetchdir, urldir)
if not repodir in done_repos:
if not lcommit:
lcommit = utils.runcmd(['git', 'rev-list', '-1', '--before=%s' % lcommitdate, 'origin/master'], repodir, logger=logger).strip()
if lb.layer.name == 'openembedded-core':
lmapcommit = oecore_map.get(lcommit, None)
if lmapcommit:
logger.debug('Preferring OE-Core revision %s over %s' % (lmapcommit, lcommit))
lcommit = lmapcommit
utils.checkout_repo(repodir, lcommit, logger, force)
if lcommit in oecore_bad_revs_2:
# Fix issue that was introduced in 309a02931779f32d1139cc1169a039cbe4638706 and fixed in 40a904bf8bc1279c3da0893c003f740f1d2066c2
with open(os.path.join(repodir, 'meta/conf/bitbake.conf'), 'r') as f:
lines = f.readlines()
lines.insert(0, 'BBINCLUDED ?= ""\n')
with open(os.path.join(repodir, 'meta/conf/bitbake.conf'), 'w') as f:
f.writelines(lines)
done_repos.append(repodir)
# We "force" here because it's almost certain we'll be checking out a
# different revision for the layer itself
checkout_layer(layerbranch, commit, force=True)
layer_urldir = str(layerbranch.layer.get_fetch_dir())
layer_repodir = os.path.join(fetchdir, layer_urldir)
commitdate = utils.runcmd(['git', 'show', '-s', '--format=%ci'], layer_repodir, logger=logger)
for dep in layerbranch.get_recursive_dependencies():
checkout_layer(dep, lcommitdate=commitdate)
return commitdate
def checkout_layer(lb, lcommit=None, lcommitdate=None, force=False):
urldir = str(lb.layer.get_fetch_dir())
repodir = os.path.join(fetchdir, urldir)
if not repodir in done_repos:
if not lcommit:
lcommit = utils.runcmd(['git', 'rev-list', '-1', '--before=%s' % lcommitdate, 'origin/master'], repodir, logger=logger).strip()
if lb.layer.name == 'openembedded-core':
lmapcommit = oecore_map.get(lcommit, None)
if lmapcommit:
logger.debug('Preferring OE-Core revision %s over %s' % (lmapcommit, lcommit))
lcommit = lmapcommit
utils.checkout_repo(repodir, lcommit, logger, force)
if lcommit in oecore_bad_revs_2:
# Fix issue that was introduced in 309a02931779f32d1139cc1169a039cbe4638706 and fixed in 40a904bf8bc1279c3da0893c003f740f1d2066c2
with open(os.path.join(repodir, 'meta/conf/bitbake.conf'), 'r') as f:
lines = f.readlines()
lines.insert(0, 'BBINCLUDED ?= ""\n')
with open(os.path.join(repodir, 'meta/conf/bitbake.conf'), 'w') as f:
f.writelines(lines)
done_repos.append(repodir)
def maintainers_inc_history(options, logger, maintplan, layerbranch, repodir,
layerdir):
maintainers_full_path = os.path.join(layerdir, MAINTAINERS_INCLUDE_PATH)
if not os.path.exists(maintainers_full_path):
logger.warning(
'Maintainer style is maintainers.inc for plan %s but no maintainers.inc exists in for %s'
% (maintplan, layerbranch))
return
logger.debug('Checking maintainers.inc history for %s' % layerbranch)
commits = utils.runcmd(
"git log --format='%%H' --reverse --date=rfc origin/master %s" %
os.path.join(layerbranch.vcs_subdir, MAINTAINERS_INCLUDE_PATH),
repodir,
logger=logger)
no_maintainer, _ = Maintainer.objects.get_or_create(name='No maintainer')
try:
with transaction.atomic():
for commit in commits.strip().split("\n"):
if RecipeMaintainerHistory.objects.filter(
layerbranch=layerbranch, sha1=commit):
continue
logger.debug("Analysing commit %s ..." % (commit))
(author_name, author_email, date, title) = \
get_commit_info(utils.runcmd("git show " + commit, repodir,
logger=logger), logger)
author = Maintainer.create_or_update(author_name, author_email)
rms = RecipeMaintainerHistory(title=title,
date=date,
author=author,
sha1=commit,
layerbranch=layerbranch)
rms.save()
utils.runcmd("git checkout %s -f" % commit,
repodir,
logger=logger)
lines = [line.strip() for line in open(maintainers_full_path)]
for line in lines:
res = get_recipe_maintainer(line, logger)
if res:
(pn, name, email) = res
qry = Recipe.objects.filter(pn=pn,
layerbranch=layerbranch)
if qry:
m = Maintainer.create_or_update(name, email)
rm = RecipeMaintainer()
rm.recipe = qry[0]
rm.maintainer = m
rm.history = rms
rm.save()
logger.debug("%s: Change maintainer to %s in commit %s." % \
(pn, m.name, commit))
else:
logger.debug("%s: Not found in %s." % \
(pn, layerbranch))
# set missing recipes to no maintainer
for recipe in layerbranch.recipe_set.all():
if not RecipeMaintainer.objects.filter(recipe=recipe,
history=rms):
rm = RecipeMaintainer()
rm.recipe = recipe
link_maintainer = RecipeMaintenanceLink.link_maintainer(
recipe.pn, rms)
if link_maintainer:
rm.maintainer = link_maintainer.maintainer
else:
rm.maintainer = no_maintainer
rm.history = rms
rm.save()
if link_maintainer:
logger.debug(
"%s: linked to maintainer for %s" %
(recipe.pn, link_maintainer.recipe.pn))
else:
logger.debug("%s: Not found maintainer in commit %s set to 'No maintainer'." % \
(recipe.pn, rms.sha1))
# set new recipes to no maintainer if don't have one
rms = RecipeMaintainerHistory.get_last(layerbranch)
for recipe in layerbranch.recipe_set.all():
if not RecipeMaintainer.objects.filter(recipe=recipe,
history=rms):
rm = RecipeMaintainer()
rm.recipe = recipe
link_maintainer = RecipeMaintenanceLink.link_maintainer(
recipe.pn, rms)
if link_maintainer:
rm.maintainer = link_maintainer.maintainer
else:
rm.maintainer = no_maintainer
rm.history = rms
rm.save()
if link_maintainer:
logger.debug(
"%s: New recipe linked to maintainer for %s" %
(recipe.pn, link_maintainer.recipe.pn))
else:
logger.debug("%s: New recipe not found maintainer set to 'No maintainer'." % \
(recipe.pn))
if options.dry_run:
raise DryRunRollbackException
except DryRunRollbackException:
pass
def generate_history(options, layerbranch_id, commit, logger):
from layerindex.models import LayerBranch
from rrs.models import Release
layerbranch = LayerBranch.objects.get(id=layerbranch_id)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
layer = layerbranch.layer
urldir = str(layer.get_fetch_dir())
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, str(layerbranch.vcs_subdir))
commitdate = checkout_layer_deps(layerbranch, commit, fetchdir, logger)
if options.initial:
fns = None
else:
fns = _get_recipes_filenames(commit, repodir, layerdir, logger)
if not fns:
return
# setup bitbake
bitbakepath = os.path.join(fetchdir, 'bitbake')
if options.bitbake_rev:
bitbake_rev = options.bitbake_rev
if not re.match('^[0-9a-f]{40}$', bitbake_rev):
# Branch name, need to check out detached
bitbake_rev = 'origin/%s' % bitbake_rev
else:
bitbake_rev = utils.runcmd([
'git', 'rev-list', '-1',
'--before=%s' % commitdate, 'origin/master'
],
bitbakepath,
logger=logger).strip()
utils.checkout_repo(bitbakepath, bitbake_rev, logger)
sys.path.insert(0, os.path.join(bitbakepath, 'lib'))
(tinfoil, d, recipes, tempdir) = load_recipes(layerbranch,
bitbakepath,
fetchdir,
settings,
logger,
recipe_files=fns,
nocheckout=True)
try:
if options.initial:
title = options.initial
info = 'No maintainer;;' + utils.runcmd([
'git', 'log', '--format=%ad;%cd', '--date=rfc', '-n', '1',
commit
],
destdir=repodir,
logger=logger)
recordcommit = ''
else:
title = utils.runcmd(
['git', 'log', '--format=%s', '-n', '1', commit],
repodir,
logger=logger)
info = utils.runcmd([
'git', 'log', '--format=%an;%ae;%ad;%cd', '--date=rfc', '-n',
'1', commit
],
destdir=repodir,
logger=logger)
recordcommit = commit
try:
with transaction.atomic():
for recipe_data in recipes:
_create_upgrade(recipe_data,
layerbranch,
recordcommit,
title,
info,
logger,
initial=options.initial)
if options.dry_run:
raise DryRunRollbackException
except DryRunRollbackException:
pass
finally:
if tinfoil and hasattr(tinfoil, 'shutdown') and (LooseVersion(
bb.__version__) > LooseVersion("1.27")):
tinfoil.shutdown()
utils.rmtree_force(tempdir)
def upgrade_history(options, logger):
from rrs.models import MaintenancePlan, RecipeUpgrade, Release, Milestone
if options.plan:
maintplans = MaintenancePlan.objects.filter(id=int(options.plan))
if not maintplans.exists():
logger.error('No maintenance plan with ID %s found' % options.plan)
sys.exit(1)
else:
maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
if not maintplans.exists():
logger.error('No enabled maintenance plans found')
sys.exit(1)
lockfn = os.path.join(fetchdir, "layerindex.lock")
lockfile = utils.lock_file(lockfn)
if not lockfile:
logger.error("Layer index lock timeout expired")
sys.exit(1)
try:
for maintplan in maintplans:
# Check releases and milestones
current = date.today()
current_release = Release.get_by_date(maintplan, current)
if current_release:
current_milestone = Milestone.get_by_release_and_date(
current_release, current)
if not current_milestone:
logger.warning(
'%s: there is no milestone defined in the latest release (%s) that covers the current date, so up-to-date data will not be visible in the web interface.'
% (maintplan, current_release))
else:
logger.warning(
'%s: there is no release defined that covers the current date, so up-to-date data will not be visible in the web interface.'
% maintplan)
for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all(
):
layerbranch = maintplanbranch.layerbranch
if options.fullreload and not options.dry_run:
RecipeUpgrade.objects.filter(
recipe__layerbranch=layerbranch).delete()
layer = layerbranch.layer
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
if options.commit:
initial = False
since = options.commit
since_option = '%s^..%s' % (options.commit, options.commit)
elif maintplanbranch.upgrade_rev and not options.fullreload:
initial = False
since = maintplanbranch.upgrade_date
since_option = '%s..origin/master' % maintplanbranch.upgrade_rev
else:
initial = True
since = options.since
since_option = '--since="%s" origin/master' % since
repo = git.Repo(repodir)
assert repo.bare == False
commits = utils.runcmd(
"git log %s --format='%%H %%ct' --reverse" % since_option,
repodir,
logger=logger)
commit_list = commits.split('\n')
bitbake_map = {}
# Filter out some bad commits
bitbake_commits = utils.runcmd(
"git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^",
bitbakepath,
logger=logger)
bitbake_commit_list = bitbake_commits.splitlines()
for commit in bitbake_commit_list:
bitbake_map[
commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d'
if initial:
logger.debug("Adding initial upgrade history ....")
ct, ctepoch = commit_list.pop(0).split()
ctdate = datetime.fromtimestamp(int(ctepoch))
run_internal(maintplanbranch,
ct,
ctdate,
options,
logger,
bitbake_map,
initial=True)
if layerbranch.vcs_subdir:
layersubdir_start = layerbranch.vcs_subdir
if not layersubdir_start.endswith('/'):
layersubdir_start += '/'
else:
layersubdir_start = ''
logger.debug("Adding upgrade history from %s to %s ..." %
(since, datetime.today().strftime("%Y-%m-%d")))
for item in commit_list:
if item:
ct, ctepoch = item.split()
ctdate = datetime.fromtimestamp(int(ctepoch))
commitobj = repo.commit(ct)
touches_recipe = False
for parent in commitobj.parents:
diff = parent.diff(commitobj)
for diffitem in diff:
if layersubdir_start and not (
diffitem.a_path.startswith(
layersubdir_start) or diffitem.
b_path.startswith(layersubdir_start)):
# Not in this layer, skip it
continue
if diffitem.a_path.endswith(
('.bb',
'.inc')) or diffitem.b_path.endswith(
('.bb', '.inc')):
# We need to look at this commit
touches_recipe = True
break
if touches_recipe:
break
if not touches_recipe:
# No recipes in the layer changed in this commit
# NOTE: Whilst it's possible that a change to a class might alter what's
# in the recipe, we can ignore that since we are only concerned with actual
# upgrades which would always require some sort of change to the recipe
# or an include file, so we can safely skip commits that don't do that
logger.debug("Skipping commit %s" % ct)
continue
logger.debug("Analysing commit %s ..." % ct)
run_internal(maintplanbranch, ct, ctdate, options,
logger, bitbake_map)
if not options.dry_run:
maintplanbranch.upgrade_rev = ct
maintplanbranch.upgrade_date = ctdate
maintplanbranch.save()
finally:
utils.unlock_file(lockfile)
def send_email(maintplan, recipes, options):
upgradable_count = 0
no_upgradable_count = 0
maintainers = Maintainer.objects.all().order_by("name")
RecipeUpgradeLine = namedtuple(
'RecipeUpgradeLine',
['pn', 'pv', 'pv_upstream', 'maintainer', 'noupgradereason'])
recipelines = []
for m in maintainers:
for recipe in recipes.keys():
recipe_maintainer = recipes[recipe]['maintainer']
recipe_upstream = recipes[recipe]['upstream']
if m.id == recipe_maintainer.maintainer.id:
pn_max_len = 20
pv_max_len = 20
name_max_len = 20
reason_max_len = 30
recipelines.append(
RecipeUpgradeLine(recipe.pn, recipe.pv,
recipe_upstream.version, m.name,
recipe_upstream.no_update_reason))
upgradable_count = upgradable_count + 1
if recipe_upstream.no_update_reason:
no_upgradable_count = no_upgradable_count + 1
commits = []
fetchdir = settings.LAYER_FETCH_DIR
for item in maintplan.maintenanceplanlayerbranch_set.all():
layerbranch = item.layerbranch
layer = layerbranch.layer
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
# FIXME this assumes the correct branch is checked out
topcommitdesc = utils.runcmd("git log -1 --oneline", repodir).strip()
commits.append('%s: %s' % (layerbranch.layer.name, topcommitdesc))
# Render the subject as a template (to allow a bit of flexibility)
subject = options.subject or maintplan.email_subject
subject_template = Template(subject)
current_site = Site.objects.get_current()
d = Context({
'maintplan': maintplan,
'site': current_site,
})
subject_content = subject_template.render(d)
from_email = options._from or maintplan.email_from
if options.to:
to_email_list = options.to.split(';')
elif maintplan.email_to:
to_email_list = maintplan.email_to.split(';')
else:
to_email_list = []
if not subject:
logger.error(
'No subject specified in maintenance plan %s and none specified on command line'
% maintplan.name)
sys.exit(1)
if not from_email:
logger.error(
'No sender email address specified in maintenance plan %s and none specified on command line'
% maintplan.name)
sys.exit(1)
if not to_email_list:
logger.error(
'No recipient email address specified in maintenance plan %s and none specified on command line'
% maintplan.name)
sys.exit(1)
plaintext = get_template('rrs/report_email.txt')
site_url = 'http://' + current_site.domain + reverse('rrs_frontpage')
d = {
'maintplan': maintplan,
'site': current_site,
'site_url': site_url,
'upgradable_count': (upgradable_count - no_upgradable_count),
'total_upgradable_count': upgradable_count,
'commits': commits,
'recipelines': recipelines,
}
text_content = plaintext.render(d)
msg = EmailMessage(subject_content, text_content, from_email,
to_email_list)
msg.send()
def upgrade_history(options, logger):
from rrs.models import MaintenancePlan, RecipeUpgrade, Release, Milestone, RecipeUpgradeGroupRule
logger.debug('=== BEGIN; cmdline: %s' % (' '.join(sys.argv)))
if options.plan:
maintplans = MaintenancePlan.objects.filter(id=int(options.plan))
if not maintplans.exists():
logger.error('No maintenance plan with ID %s found' % options.plan)
sys.exit(1)
else:
maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
if not maintplans.exists():
logger.error('No enabled maintenance plans found')
sys.exit(1)
if options.regroup:
for maintplan in maintplans:
for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all(
):
layerbranch = maintplanbranch.layerbranch
if RecipeUpgradeGroupRule.objects.filter(
layerbranch=layerbranch).exists():
for ru in RecipeUpgrade.objects.filter(
recipesymbol__layerbranch=layerbranch):
if ru.regroup():
ru.save()
sys.exit(0)
lockfn = os.path.join(fetchdir, "layerindex.lock")
lockfile = utils.lock_file(lockfn)
if not lockfile:
logger.error("Layer index lock timeout expired")
sys.exit(1)
try:
for maintplan in maintplans:
# Check releases and milestones
current = date.today()
current_release = Release.get_by_date(maintplan, current)
if current_release:
current_milestone = Milestone.get_by_release_and_date(
current_release, current)
if not current_milestone:
logger.warning(
'%s: there is no milestone defined in the latest release (%s) that covers the current date, so up-to-date data will not be visible in the web interface.'
% (maintplan, current_release))
else:
logger.warning(
'%s: there is no release defined that covers the current date, so up-to-date data will not be visible in the web interface.'
% maintplan)
for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all(
):
layerbranch = maintplanbranch.layerbranch
if options.fullreload and not options.dry_run:
logger.debug('fullreload: deleting upgrade objects')
if options.filter_files:
RecipeUpgrade.objects.filter(
recipesymbol__layerbranch=layerbranch,
filepath__startswith=options.filter_files).delete(
)
else:
RecipeUpgrade.objects.filter(
recipesymbol__layerbranch=layerbranch).delete()
layer = layerbranch.layer
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
if options.commit:
initial = False
since = options.commit
since_option = [
'%s^..%s' % (options.commit, options.commit)
]
elif maintplanbranch.upgrade_rev and not options.fullreload:
initial = False
since = maintplanbranch.upgrade_date
since_option = [
'%s..origin/master' % maintplanbranch.upgrade_rev
]
else:
initial = True
since = options.since
since_option = ['--since=%s' % since, 'origin/master']
repo = git.Repo(repodir)
if repo.bare:
logger.error('Repository %s is bare, not supported' %
repodir)
continue
commits = utils.runcmd(['git', 'log'] + since_option +
['--format=%H %ct', '--reverse'],
repodir,
logger=logger)
commit_list = commits.split('\n')
bitbake_map = {}
def remap_range(start, end, replacewith=None):
if replacewith is None:
replacewith = end
bitbake_commits = utils.runcmd(
['git', 'rev-list',
'%s^..%s^' % (start, end)],
bitbakepath,
logger=logger)
bitbake_commit_list = bitbake_commits.splitlines()
for commit in bitbake_commit_list:
bitbake_map[commit] = replacewith
# Filter out some bad commits
remap_range('fef18b445c0cb6b266cd939b9c78d7cbce38663f',
'39780b1ccbd76579db0fc6fb9369c848a3bafa9d')
remap_range('5796ed550d127853808f38257f8dcc8c1cf59342',
'547128731e62b36d2271c4390b3fee2b16c535dc')
if options.stop_commit and (options.stop_commit not in [
x.split()[0] for x in commit_list
]):
logger.error('Stop commit %s is not in repository %s' %
(options.stop_commit, repodir))
sys.exit(1)
if initial:
logger.debug("Adding initial upgrade history ....")
ct, ctepoch = commit_list.pop(0).split()
ctdate = datetime.fromtimestamp(int(ctepoch))
run_internal(maintplanbranch,
ct,
ctdate,
options,
logger,
bitbake_map,
initial=True)
if layerbranch.vcs_subdir:
layersubdir_start = layerbranch.vcs_subdir
if not layersubdir_start.endswith('/'):
layersubdir_start += '/'
else:
layersubdir_start = ''
logger.debug("Adding upgrade history from %s to %s ..." %
(since, datetime.today().strftime("%Y-%m-%d")))
for item in commit_list:
if item:
ct, ctepoch = item.split()
if ct == options.stop_commit:
logger.debug('Stopping at requested commit %s' %
ct)
break
ctdate = datetime.fromtimestamp(int(ctepoch))
commitobj = repo.commit(ct)
touches_recipe = False
for parent in commitobj.parents:
diff = parent.diff(commitobj)
for diffitem in diff:
if layersubdir_start and not (
diffitem.a_path.startswith(
layersubdir_start) or diffitem.
b_path.startswith(layersubdir_start)):
# Not in this layer, skip it
continue
if options.filter_files and not (
diffitem.a_path.startswith(
options.filter_files)
or diffitem.b_path.startswith(
options.filter_files)):
# Doesn't match path filter
continue
if diffitem.a_path.endswith(
('.bb',
'.inc')) or diffitem.b_path.endswith(
('.bb', '.inc')):
# We need to look at this commit
touches_recipe = True
break
if touches_recipe:
break
if not touches_recipe:
# No recipes in the layer changed in this commit
# NOTE: Whilst it's possible that a change to a class might alter what's
# in the recipe, we can ignore that since we are only concerned with actual
# upgrades which would always require some sort of change to the recipe
# or an include file, so we can safely skip commits that don't do that
logger.debug("Skipping commit %s" % ct)
continue
logger.debug("Analysing commit %s ..." % ct)
run_internal(maintplanbranch, ct, ctdate, options,
logger, bitbake_map)
if not (options.dry_run or options.filter_files):
maintplanbranch.upgrade_rev = ct
maintplanbranch.upgrade_date = ctdate
maintplanbranch.save()
finally:
utils.unlock_file(lockfile)
def generate_history(options, layerbranch_id, commit, logger):
from layerindex.models import LayerBranch
from rrs.models import Release, RecipeUpgrade
layerbranch = LayerBranch.objects.get(id=layerbranch_id)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
layer = layerbranch.layer
urldir = str(layer.get_fetch_dir())
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, str(layerbranch.vcs_subdir))
if layerbranch.vcs_subdir:
layersubdir_start = layerbranch.vcs_subdir
if not layersubdir_start.endswith('/'):
layersubdir_start += '/'
else:
layersubdir_start = ''
repo = git.Repo(repodir)
if repo.bare:
logger.error('Repository %s is bare, not supported' % repodir)
sys.exit(1)
commitdate = checkout_layer_deps(layerbranch, commit, fetchdir, logger)
if options.initial:
fns = None
deleted = []
moved = []
else:
if options.filter_files:
filepath_start = options.filter_files
else:
filepath_start = layersubdir_start
fns, deleted, moved = _get_recipes_filenames(commit, repo, repodir, filepath_start, logger)
if not (fns or deleted or moved):
return
# setup bitbake
bitbakepath = os.path.join(fetchdir, 'bitbake')
if options.bitbake_rev:
bitbake_rev = options.bitbake_rev
if not re.match('^[0-9a-f]{40}$', bitbake_rev):
# Branch name, need to check out detached
bitbake_rev = 'origin/%s' % bitbake_rev
else:
bitbake_rev = utils.runcmd(['git', 'rev-list', '-1', '--before=%s' % commitdate, 'origin/master'], bitbakepath, logger=logger).strip()
utils.checkout_repo(bitbakepath, bitbake_rev, logger)
sys.path.insert(0, os.path.join(bitbakepath, 'lib'))
(tinfoil, d, recipes, tempdir) = load_recipes(layerbranch, bitbakepath,
fetchdir, settings, logger, recipe_files=fns,
nocheckout=True)
try:
if options.initial:
title = options.initial
info = 'No maintainer;;' + utils.runcmd(['git', 'log', '--format=%ad;%cd', '--date=rfc', '-n', '1', commit], destdir=repodir, logger=logger)
recordcommit = ''
else:
title = utils.runcmd(['git', 'log', '--format=%s', '-n', '1', commit],
repodir, logger=logger)
info = utils.runcmd(['git', 'log', '--format=%an;%ae;%ad;%cd', '--date=rfc', '-n', '1', commit], destdir=repodir, logger=logger)
recordcommit = commit
fn_data = {}
for recipe_data in recipes:
fn = os.path.relpath(recipe_data.getVar('FILE', True), repodir)
fn_data[fn] = recipe_data
seen_pns = []
try:
with transaction.atomic():
# Handle recipes where PN has changed
for a, b in moved:
logger.debug('Move %s -> %s' % (a,b))
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date', '-id')
recipe_data = fn_data.get(b, None)
if recipe_data:
pn = recipe_data.getVar('PN', True)
ru = rus.first()
if ru and ru.recipesymbol.pn != pn:
# PN has been changed! We need to mark the old record as deleted
logger.debug('PN changed (with move): %s -> %s' % (ru.recipesymbol.pn, pn))
if a not in deleted:
deleted.append(a)
else:
logger.warning('Unable to find parsed data for recipe %s' % b)
# Handle recipes that exist at this point in time (which may have upgraded)
for recipe_data in recipes:
pn = recipe_data.getVar('PN', True)
filepath = os.path.relpath(recipe_data.getVar('FILE', True), repodir)
# Check if PN has changed internally
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=filepath).order_by('-commit_date', '-id')
deleted_pns = rus.filter(upgrade_type__in=['R', 'N']).values_list('recipesymbol__pn', flat=True).distinct()
for ru in rus:
if ru.recipesymbol.pn != pn and ru.recipesymbol.pn not in deleted_pns and ru.upgrade_type not in ['R', 'N']:
# PN changed (set within recipe), we need to mark the old recipe as deleted
logger.debug('PN changed (without move): %s -> %s' % (ru.recipesymbol.pn, pn))
_save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, ru.license, recordcommit, title, info, ru.filepath, logger, upgrade_type='R')
orig_filepath = None
for a, b in moved:
if b == filepath:
orig_filepath = a
break
_create_upgrade(recipe_data, layerbranch, recordcommit, title,
info, filepath, logger, initial=options.initial, orig_filepath=orig_filepath)
seen_pns.append(pn)
# Handle recipes that have been moved without it being an upgrade/delete
for a, b in moved:
if a not in deleted:
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=a).order_by('-commit_date', '-id')
if rus:
ru = rus.first()
if not RecipeUpgrade.objects.filter(recipesymbol=ru.recipesymbol, filepath=b).exists():
# Need to record the move, otherwise we won't be able to
# find the record if we need to mark the recipe as deleted later
_save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, ru.license, recordcommit, title, info, b, logger, upgrade_type='M', orig_filepath=a)
# Handle deleted recipes
for df in deleted:
rus = RecipeUpgrade.objects.filter(recipesymbol__layerbranch=layerbranch, filepath=df).order_by('-commit_date', '-id')
for ru in rus:
other_rus = RecipeUpgrade.objects.filter(recipesymbol=ru.recipesymbol, commit_date__gte=ru.commit_date).exclude(filepath=df).order_by('-commit_date', '-id')
# We make a distinction between deleting just one version and the entire recipe being deleted
upgrade_type = 'R'
for other_ru in other_rus:
if other_ru.upgrade_type == 'R':
logger.debug('There is a delete: %s' % other_ru)
upgrade_type = ''
break
if os.path.exists(os.path.join(repodir, other_ru.filepath)):
upgrade_type = 'N'
if not upgrade_type:
continue
if ru.upgrade_type != upgrade_type and ru.recipesymbol.pn not in seen_pns:
if upgrade_type == 'R':
finalmsg = ' [FINAL]'
else:
finalmsg = ''
logger.debug("%s: marking as deleted%s (%s)" % (ru.recipesymbol.pn, finalmsg, ru.filepath))
_save_upgrade(ru.recipesymbol, layerbranch, ru.version, ru.srcrev, ru.license, recordcommit, title, info, df, logger, upgrade_type=upgrade_type)
break
if options.dry_run:
raise DryRunRollbackException
except DryRunRollbackException:
pass
finally:
if tinfoil and hasattr(tinfoil, 'shutdown') and (LooseVersion(bb.__version__) > LooseVersion("1.27")):
tinfoil.shutdown()
utils.rmtree_force(tempdir)