def run_internal(maintplanlayerbranch,
commit,
commitdate,
options,
logger,
bitbake_map,
initial=False):
from layerindex.models import PythonEnvironment
from rrs.models import Release
if commitdate < maintplanlayerbranch.python3_switch_date:
# Python 2
if maintplanlayerbranch.python2_environment:
cmdprefix = maintplanlayerbranch.python2_environment.get_command()
else:
cmdprefix = 'python'
# Ensure we're using a bitbake version that is python 2 compatible
if commitdate > datetime(2016, 5, 10):
commitdate = datetime(2016, 5, 10)
else:
# Python 3
if maintplanlayerbranch.python3_environment:
cmdprefix = maintplanlayerbranch.python3_environment.get_command()
else:
cmdprefix = 'python3'
bitbake_rev = utils.runcmd([
'git', 'rev-list', '-1',
'--before=%s' % str(commitdate), 'origin/master'
],
bitbakepath,
logger=logger)
check_rev = bitbake_map.get(bitbake_rev, None)
if check_rev:
logger.debug('Preferring bitbake revision %s over %s' %
(check_rev, bitbake_rev))
bitbake_rev = check_rev
cmd = '%s upgrade_history_internal.py %s %s' % (
cmdprefix, maintplanlayerbranch.layerbranch.id, commit)
if initial:
release = Release.get_by_date(maintplanlayerbranch.plan, commitdate)
if release:
comment = 'Initial import at %s release start.' % release.name
else:
comment = 'Initial import at %s' % commit
cmd += ' --initial="%s"' % comment
if bitbake_rev:
cmd += ' --bitbake-rev %s' % bitbake_rev
if options.filter_files:
cmd += ' --filter-files %s' % options.filter_files
if options.dry_run:
cmd += ' --dry-run'
if options.loglevel == logging.DEBUG:
cmd += ' --debug'
logger.debug('Running %s' % cmd)
ret, output = utils.run_command_interruptible(cmd)
if ret == 254:
# Interrupted by user, break out of loop
logger.info('Update interrupted, exiting')
sys.exit(254)
def _get_recipe_upgrade_detail(maintplan, recipe_upgrade):
release_name = ''
milestone_name = ''
recipe_maintainer_history = None
release = Release.get_by_date(maintplan, recipe_upgrade.commit_date)
if release:
release_name = release.name
milestone = Milestone.get_by_release_and_date(
release, recipe_upgrade.commit_date)
if milestone:
milestone_name = milestone.name
recipe_maintainer_history = RecipeMaintainerHistory.get_by_end_date(
recipe_upgrade.recipesymbol.layerbranch, milestone.end_date)
is_recipe_maintainer = False
maintainer_name = ''
if not recipe_upgrade.maintainer is None:
maintainer_name = recipe_upgrade.maintainer.name
if not recipe_maintainer_history is None and \
RecipeMaintainer.objects.filter(maintainer__name
= maintainer_name, history = recipe_maintainer_history) \
.count() > 0:
is_recipe_maintainer = True
commit_date = recipe_upgrade.commit_date.date().isoformat()
commit = recipe_upgrade.sha1[:10]
commit_url = recipe_upgrade.recipesymbol.layerbranch.commit_url(
recipe_upgrade.sha1)
rud = RecipeUpgradeDetail(recipe_upgrade.title, recipe_upgrade.version, \
maintplan.name, release_name, milestone_name, commit_date, maintainer_name, \
is_recipe_maintainer, commit, commit_url, recipe_upgrade.upgrade_type, \
recipe_upgrade.group)
return rud
def upgrade_history(options, logger):
from rrs.models import MaintenancePlan, RecipeUpgrade, Release, Milestone
if options.plan:
maintplans = MaintenancePlan.objects.filter(id=int(options.plan))
if not maintplans.exists():
logger.error('No maintenance plan with ID %s found' % options.plan)
sys.exit(1)
else:
maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
if not maintplans.exists():
logger.error('No enabled maintenance plans found')
sys.exit(1)
lockfn = os.path.join(fetchdir, "layerindex.lock")
lockfile = utils.lock_file(lockfn)
if not lockfile:
logger.error("Layer index lock timeout expired")
sys.exit(1)
try:
for maintplan in maintplans:
# Check releases and milestones
current = date.today()
current_release = Release.get_by_date(maintplan, current)
if current_release:
current_milestone = Milestone.get_by_release_and_date(
current_release, current)
if not current_milestone:
logger.warning(
'%s: there is no milestone defined in the latest release (%s) that covers the current date, so up-to-date data will not be visible in the web interface.'
% (maintplan, current_release))
else:
logger.warning(
'%s: there is no release defined that covers the current date, so up-to-date data will not be visible in the web interface.'
% maintplan)
for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all(
):
layerbranch = maintplanbranch.layerbranch
if options.fullreload and not options.dry_run:
RecipeUpgrade.objects.filter(
recipe__layerbranch=layerbranch).delete()
layer = layerbranch.layer
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
if options.commit:
initial = False
since = options.commit
since_option = '%s^..%s' % (options.commit, options.commit)
elif maintplanbranch.upgrade_rev and not options.fullreload:
initial = False
since = maintplanbranch.upgrade_date
since_option = '%s..origin/master' % maintplanbranch.upgrade_rev
else:
initial = True
since = options.since
since_option = '--since="%s" origin/master' % since
repo = git.Repo(repodir)
assert repo.bare == False
commits = utils.runcmd(
"git log %s --format='%%H %%ct' --reverse" % since_option,
repodir,
logger=logger)
commit_list = commits.split('\n')
bitbake_map = {}
# Filter out some bad commits
bitbake_commits = utils.runcmd(
"git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^",
bitbakepath,
logger=logger)
bitbake_commit_list = bitbake_commits.splitlines()
for commit in bitbake_commit_list:
bitbake_map[
commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d'
if initial:
logger.debug("Adding initial upgrade history ....")
ct, ctepoch = commit_list.pop(0).split()
ctdate = datetime.fromtimestamp(int(ctepoch))
run_internal(maintplanbranch,
ct,
ctdate,
options,
logger,
bitbake_map,
initial=True)
if layerbranch.vcs_subdir:
layersubdir_start = layerbranch.vcs_subdir
if not layersubdir_start.endswith('/'):
layersubdir_start += '/'
else:
layersubdir_start = ''
logger.debug("Adding upgrade history from %s to %s ..." %
(since, datetime.today().strftime("%Y-%m-%d")))
for item in commit_list:
if item:
ct, ctepoch = item.split()
ctdate = datetime.fromtimestamp(int(ctepoch))
commitobj = repo.commit(ct)
touches_recipe = False
for parent in commitobj.parents:
diff = parent.diff(commitobj)
for diffitem in diff:
if layersubdir_start and not (
diffitem.a_path.startswith(
layersubdir_start) or diffitem.
b_path.startswith(layersubdir_start)):
# Not in this layer, skip it
continue
if diffitem.a_path.endswith(
('.bb',
'.inc')) or diffitem.b_path.endswith(
('.bb', '.inc')):
# We need to look at this commit
touches_recipe = True
break
if touches_recipe:
break
if not touches_recipe:
# No recipes in the layer changed in this commit
# NOTE: Whilst it's possible that a change to a class might alter what's
# in the recipe, we can ignore that since we are only concerned with actual
# upgrades which would always require some sort of change to the recipe
# or an include file, so we can safely skip commits that don't do that
logger.debug("Skipping commit %s" % ct)
continue
logger.debug("Analysing commit %s ..." % ct)
run_internal(maintplanbranch, ct, ctdate, options,
logger, bitbake_map)
if not options.dry_run:
maintplanbranch.upgrade_rev = ct
maintplanbranch.upgrade_date = ctdate
maintplanbranch.save()
finally:
utils.unlock_file(lockfile)
def get_context_data(self, **kwargs):
context = super(RecipeListView, self).get_context_data(**kwargs)
context['this_url_name'] = resolve(self.request.path_info).url_name
context['all_maintplans'] = MaintenancePlan.objects.all()
context['maintplan_name'] = self.maintplan_name
maintplan = get_object_or_404(MaintenancePlan,
name=self.maintplan_name)
context['maintplan'] = maintplan
context['release_name'] = self.release_name
context['all_releases'] = Release.objects.filter(
plan=maintplan).order_by('-end_date')
context['milestone_name'] = self.milestone_name
context['all_milestones'] = Milestone.get_by_release_name(
maintplan, self.release_name)
current = date.today()
current_release = Release.get_by_date(maintplan, current)
if current_release:
current_milestone = Milestone.get_by_release_and_date(
current_release, current)
if not current_milestone:
messages.error(
self.request,
'There is no milestone defined in the latest release (%s) that covers the current date, so data shown here is not up-to-date. The administrator will need to create a milestone in order to fix this.'
% current_release)
else:
messages.error(
self.request,
'There is no release defined that covers the current date, so data shown here is not up-to-date. The administrator will need to create a release (and corresponding milestones) in order to fix this.'
)
context['recipes_percentage'] = self.milestone_statistics['percentage']
context['recipes_all_upgraded'] = self.milestone_statistics[
'all_upgraded']
context['recipes_all_not_upgraded'] = self.milestone_statistics[
'all_not_upgraded']
context['recipes_up_to_date'] = self.milestone_statistics['up_to_date']
context['recipes_not_updated'] = self.milestone_statistics[
'not_updated']
context['recipes_cant_be_updated'] = self.milestone_statistics[
'cant_be_updated']
context['recipes_unknown'] = self.milestone_statistics['unknown']
context['recipes_percentage_up_to_date'] = \
self.milestone_statistics['percentage_up_to_date']
context['recipes_percentage_not_updated'] = \
self.milestone_statistics['percentage_not_updated']
context['recipes_percentage_cant_be_updated'] = \
self.milestone_statistics['percentage_cant_be_updated']
context['recipes_percentage_unknown'] = \
self.milestone_statistics['percentage_unknown']
context['recipe_list_count'] = self.recipe_list_count
context['upstream_status'] = self.upstream_status
ruch = RecipeUpstream.RECIPE_UPSTREAM_STATUS_CHOICES_DICT
context['upstream_status_set_choices'] = [ruch['A']]
context['upstream_status_choices'] = [
ruch['N'], ruch['C'], ruch['Y'], ruch['U']
]
context['maintainer_name'] = self.maintainer_name
context['set_maintainers'] = ['All', 'No maintainer']
all_maintainers = []
for layerbranch_id, rmh in self.recipe_maintainer_history.items():
for rm in RecipeMaintainer.objects.filter(
history=rmh).values('maintainer__name').distinct(
).order_by('maintainer__name'):
if rm['maintainer__name'] in context['set_maintainers']:
continue
all_maintainers.append(rm['maintainer__name'])
context['all_maintainers'] = all_maintainers
context['search'] = self.search
return context
def upgrade_history(options, logger):
from rrs.models import MaintenancePlan, RecipeUpgrade, Release, Milestone, RecipeUpgradeGroupRule
logger.debug('=== BEGIN; cmdline: %s' % (' '.join(sys.argv)))
if options.plan:
maintplans = MaintenancePlan.objects.filter(id=int(options.plan))
if not maintplans.exists():
logger.error('No maintenance plan with ID %s found' % options.plan)
sys.exit(1)
else:
maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
if not maintplans.exists():
logger.error('No enabled maintenance plans found')
sys.exit(1)
if options.regroup:
for maintplan in maintplans:
for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all(
):
layerbranch = maintplanbranch.layerbranch
if RecipeUpgradeGroupRule.objects.filter(
layerbranch=layerbranch).exists():
for ru in RecipeUpgrade.objects.filter(
recipesymbol__layerbranch=layerbranch):
if ru.regroup():
ru.save()
sys.exit(0)
lockfn = os.path.join(fetchdir, "layerindex.lock")
lockfile = utils.lock_file(lockfn)
if not lockfile:
logger.error("Layer index lock timeout expired")
sys.exit(1)
try:
for maintplan in maintplans:
# Check releases and milestones
current = date.today()
current_release = Release.get_by_date(maintplan, current)
if current_release:
current_milestone = Milestone.get_by_release_and_date(
current_release, current)
if not current_milestone:
logger.warning(
'%s: there is no milestone defined in the latest release (%s) that covers the current date, so up-to-date data will not be visible in the web interface.'
% (maintplan, current_release))
else:
logger.warning(
'%s: there is no release defined that covers the current date, so up-to-date data will not be visible in the web interface.'
% maintplan)
for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all(
):
layerbranch = maintplanbranch.layerbranch
if options.fullreload and not options.dry_run:
logger.debug('fullreload: deleting upgrade objects')
if options.filter_files:
RecipeUpgrade.objects.filter(
recipesymbol__layerbranch=layerbranch,
filepath__startswith=options.filter_files).delete(
)
else:
RecipeUpgrade.objects.filter(
recipesymbol__layerbranch=layerbranch).delete()
layer = layerbranch.layer
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
if options.commit:
initial = False
since = options.commit
since_option = [
'%s^..%s' % (options.commit, options.commit)
]
elif maintplanbranch.upgrade_rev and not options.fullreload:
initial = False
since = maintplanbranch.upgrade_date
since_option = [
'%s..origin/master' % maintplanbranch.upgrade_rev
]
else:
initial = True
since = options.since
since_option = ['--since=%s' % since, 'origin/master']
repo = git.Repo(repodir)
if repo.bare:
logger.error('Repository %s is bare, not supported' %
repodir)
continue
commits = utils.runcmd(['git', 'log'] + since_option +
['--format=%H %ct', '--reverse'],
repodir,
logger=logger)
commit_list = commits.split('\n')
bitbake_map = {}
def remap_range(start, end, replacewith=None):
if replacewith is None:
replacewith = end
bitbake_commits = utils.runcmd(
['git', 'rev-list',
'%s^..%s^' % (start, end)],
bitbakepath,
logger=logger)
bitbake_commit_list = bitbake_commits.splitlines()
for commit in bitbake_commit_list:
bitbake_map[commit] = replacewith
# Filter out some bad commits
remap_range('fef18b445c0cb6b266cd939b9c78d7cbce38663f',
'39780b1ccbd76579db0fc6fb9369c848a3bafa9d')
remap_range('5796ed550d127853808f38257f8dcc8c1cf59342',
'547128731e62b36d2271c4390b3fee2b16c535dc')
if options.stop_commit and (options.stop_commit not in [
x.split()[0] for x in commit_list
]):
logger.error('Stop commit %s is not in repository %s' %
(options.stop_commit, repodir))
sys.exit(1)
if initial:
logger.debug("Adding initial upgrade history ....")
ct, ctepoch = commit_list.pop(0).split()
ctdate = datetime.fromtimestamp(int(ctepoch))
run_internal(maintplanbranch,
ct,
ctdate,
options,
logger,
bitbake_map,
initial=True)
if layerbranch.vcs_subdir:
layersubdir_start = layerbranch.vcs_subdir
if not layersubdir_start.endswith('/'):
layersubdir_start += '/'
else:
layersubdir_start = ''
logger.debug("Adding upgrade history from %s to %s ..." %
(since, datetime.today().strftime("%Y-%m-%d")))
for item in commit_list:
if item:
ct, ctepoch = item.split()
if ct == options.stop_commit:
logger.debug('Stopping at requested commit %s' %
ct)
break
ctdate = datetime.fromtimestamp(int(ctepoch))
commitobj = repo.commit(ct)
touches_recipe = False
for parent in commitobj.parents:
diff = parent.diff(commitobj)
for diffitem in diff:
if layersubdir_start and not (
diffitem.a_path.startswith(
layersubdir_start) or diffitem.
b_path.startswith(layersubdir_start)):
# Not in this layer, skip it
continue
if options.filter_files and not (
diffitem.a_path.startswith(
options.filter_files)
or diffitem.b_path.startswith(
options.filter_files)):
# Doesn't match path filter
continue
if diffitem.a_path.endswith(
('.bb',
'.inc')) or diffitem.b_path.endswith(
('.bb', '.inc')):
# We need to look at this commit
touches_recipe = True
break
if touches_recipe:
break
if not touches_recipe:
# No recipes in the layer changed in this commit
# NOTE: Whilst it's possible that a change to a class might alter what's
# in the recipe, we can ignore that since we are only concerned with actual
# upgrades which would always require some sort of change to the recipe
# or an include file, so we can safely skip commits that don't do that
logger.debug("Skipping commit %s" % ct)
continue
logger.debug("Analysing commit %s ..." % ct)
run_internal(maintplanbranch, ct, ctdate, options,
logger, bitbake_map)
if not (options.dry_run or options.filter_files):
maintplanbranch.upgrade_rev = ct
maintplanbranch.upgrade_date = ctdate
maintplanbranch.save()
finally:
utils.unlock_file(lockfile)