def maintainer_history(options, logger):
fetchdir = settings.LAYER_FETCH_DIR
if options.plan:
maintplans = MaintenancePlan.objects.filter(id=int(options.plan))
if not maintplans.exists():
logger.error('No maintenance plan with ID %s found' % options.plan)
sys.exit(1)
else:
maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
if not maintplans.exists():
logger.error('No enabled maintenance plans found')
sys.exit(1)
lockfn = os.path.join(fetchdir, "layerindex.lock")
lockfile = utils.lock_file(lockfn)
if not lockfile:
logger.error("Layer index lock timeout expired")
sys.exit(1)
try:
for maintplan in maintplans:
for item in maintplan.maintenanceplanlayerbranch_set.all():
layerbranch = item.layerbranch
if options.fullreload and not options.dry_run:
RecipeMaintainerHistory.objects.filter(
layerbranch=layerbranch).delete()
urldir = str(layerbranch.layer.get_fetch_dir())
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
if maintplan.maintainer_style == 'I':
# maintainers.inc
maintainers_inc_history(options, logger, maintplan,
layerbranch, repodir, layerdir)
elif maintplan.maintainer_style == 'L':
# Layer-wide, don't need to do anything
logger.debug(
'Skipping maintainer processing for %s - plan %s maintainer style is layer-wide'
% (layerbranch, maintplan))
else:
raise Exception(
'Unknown maintainer style %s for maintenance plan %s' %
(maintplan.maintainer_style, maintplan))
finally:
utils.unlock_file(lockfile)
def upgrade_history(options, logger):
from rrs.models import MaintenancePlan, RecipeUpgrade, Release, Milestone
if options.plan:
maintplans = MaintenancePlan.objects.filter(id=int(options.plan))
if not maintplans.exists():
logger.error('No maintenance plan with ID %s found' % options.plan)
sys.exit(1)
else:
maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
if not maintplans.exists():
logger.error('No enabled maintenance plans found')
sys.exit(1)
lockfn = os.path.join(fetchdir, "layerindex.lock")
lockfile = utils.lock_file(lockfn)
if not lockfile:
logger.error("Layer index lock timeout expired")
sys.exit(1)
try:
for maintplan in maintplans:
# Check releases and milestones
current = date.today()
current_release = Release.get_by_date(maintplan, current)
if current_release:
current_milestone = Milestone.get_by_release_and_date(
current_release, current)
if not current_milestone:
logger.warning(
'%s: there is no milestone defined in the latest release (%s) that covers the current date, so up-to-date data will not be visible in the web interface.'
% (maintplan, current_release))
else:
logger.warning(
'%s: there is no release defined that covers the current date, so up-to-date data will not be visible in the web interface.'
% maintplan)
for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all(
):
layerbranch = maintplanbranch.layerbranch
if options.fullreload and not options.dry_run:
RecipeUpgrade.objects.filter(
recipe__layerbranch=layerbranch).delete()
layer = layerbranch.layer
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
if options.commit:
initial = False
since = options.commit
since_option = '%s^..%s' % (options.commit, options.commit)
elif maintplanbranch.upgrade_rev and not options.fullreload:
initial = False
since = maintplanbranch.upgrade_date
since_option = '%s..origin/master' % maintplanbranch.upgrade_rev
else:
initial = True
since = options.since
since_option = '--since="%s" origin/master' % since
repo = git.Repo(repodir)
assert repo.bare == False
commits = utils.runcmd(
"git log %s --format='%%H %%ct' --reverse" % since_option,
repodir,
logger=logger)
commit_list = commits.split('\n')
bitbake_map = {}
# Filter out some bad commits
bitbake_commits = utils.runcmd(
"git rev-list fef18b445c0cb6b266cd939b9c78d7cbce38663f^..39780b1ccbd76579db0fc6fb9369c848a3bafa9d^",
bitbakepath,
logger=logger)
bitbake_commit_list = bitbake_commits.splitlines()
for commit in bitbake_commit_list:
bitbake_map[
commit] = '39780b1ccbd76579db0fc6fb9369c848a3bafa9d'
if initial:
logger.debug("Adding initial upgrade history ....")
ct, ctepoch = commit_list.pop(0).split()
ctdate = datetime.fromtimestamp(int(ctepoch))
run_internal(maintplanbranch,
ct,
ctdate,
options,
logger,
bitbake_map,
initial=True)
if layerbranch.vcs_subdir:
layersubdir_start = layerbranch.vcs_subdir
if not layersubdir_start.endswith('/'):
layersubdir_start += '/'
else:
layersubdir_start = ''
logger.debug("Adding upgrade history from %s to %s ..." %
(since, datetime.today().strftime("%Y-%m-%d")))
for item in commit_list:
if item:
ct, ctepoch = item.split()
ctdate = datetime.fromtimestamp(int(ctepoch))
commitobj = repo.commit(ct)
touches_recipe = False
for parent in commitobj.parents:
diff = parent.diff(commitobj)
for diffitem in diff:
if layersubdir_start and not (
diffitem.a_path.startswith(
layersubdir_start) or diffitem.
b_path.startswith(layersubdir_start)):
# Not in this layer, skip it
continue
if diffitem.a_path.endswith(
('.bb',
'.inc')) or diffitem.b_path.endswith(
('.bb', '.inc')):
# We need to look at this commit
touches_recipe = True
break
if touches_recipe:
break
if not touches_recipe:
# No recipes in the layer changed in this commit
# NOTE: Whilst it's possible that a change to a class might alter what's
# in the recipe, we can ignore that since we are only concerned with actual
# upgrades which would always require some sort of change to the recipe
# or an include file, so we can safely skip commits that don't do that
logger.debug("Skipping commit %s" % ct)
continue
logger.debug("Analysing commit %s ..." % ct)
run_internal(maintplanbranch, ct, ctdate, options,
logger, bitbake_map)
if not options.dry_run:
maintplanbranch.upgrade_rev = ct
maintplanbranch.upgrade_date = ctdate
maintplanbranch.save()
finally:
utils.unlock_file(lockfile)
if options.plan:
maintplans = MaintenancePlan.objects.filter(id=int(options.plan))
if not maintplans.exists():
logger.error('No maintenance plan with ID %s found' % options.plan)
sys.exit(1)
else:
maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
if not maintplans.exists():
logger.error('No enabled maintenance plans found')
sys.exit(1)
logger.debug("Starting upstream history...")
lockfn = os.path.join(fetchdir, "layerindex.lock")
lockfile = utils.lock_file(lockfn)
if not lockfile:
logger.error("Layer index lock timeout expired")
sys.exit(1)
try:
origsyspath = sys.path
for maintplan in maintplans:
for item in maintplan.maintenanceplanlayerbranch_set.all():
layerbranch = item.layerbranch
try:
with transaction.atomic():
sys.path = origsyspath
layer = layerbranch.layer
urldir = layer.get_fetch_dir()
def upgrade_history(options, logger):
from rrs.models import MaintenancePlan, RecipeUpgrade, Release, Milestone, RecipeUpgradeGroupRule
logger.debug('=== BEGIN; cmdline: %s' % (' '.join(sys.argv)))
if options.plan:
maintplans = MaintenancePlan.objects.filter(id=int(options.plan))
if not maintplans.exists():
logger.error('No maintenance plan with ID %s found' % options.plan)
sys.exit(1)
else:
maintplans = MaintenancePlan.objects.filter(updates_enabled=True)
if not maintplans.exists():
logger.error('No enabled maintenance plans found')
sys.exit(1)
if options.regroup:
for maintplan in maintplans:
for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all(
):
layerbranch = maintplanbranch.layerbranch
if RecipeUpgradeGroupRule.objects.filter(
layerbranch=layerbranch).exists():
for ru in RecipeUpgrade.objects.filter(
recipesymbol__layerbranch=layerbranch):
if ru.regroup():
ru.save()
sys.exit(0)
lockfn = os.path.join(fetchdir, "layerindex.lock")
lockfile = utils.lock_file(lockfn)
if not lockfile:
logger.error("Layer index lock timeout expired")
sys.exit(1)
try:
for maintplan in maintplans:
# Check releases and milestones
current = date.today()
current_release = Release.get_by_date(maintplan, current)
if current_release:
current_milestone = Milestone.get_by_release_and_date(
current_release, current)
if not current_milestone:
logger.warning(
'%s: there is no milestone defined in the latest release (%s) that covers the current date, so up-to-date data will not be visible in the web interface.'
% (maintplan, current_release))
else:
logger.warning(
'%s: there is no release defined that covers the current date, so up-to-date data will not be visible in the web interface.'
% maintplan)
for maintplanbranch in maintplan.maintenanceplanlayerbranch_set.all(
):
layerbranch = maintplanbranch.layerbranch
if options.fullreload and not options.dry_run:
logger.debug('fullreload: deleting upgrade objects')
if options.filter_files:
RecipeUpgrade.objects.filter(
recipesymbol__layerbranch=layerbranch,
filepath__startswith=options.filter_files).delete(
)
else:
RecipeUpgrade.objects.filter(
recipesymbol__layerbranch=layerbranch).delete()
layer = layerbranch.layer
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
if options.commit:
initial = False
since = options.commit
since_option = [
'%s^..%s' % (options.commit, options.commit)
]
elif maintplanbranch.upgrade_rev and not options.fullreload:
initial = False
since = maintplanbranch.upgrade_date
since_option = [
'%s..origin/master' % maintplanbranch.upgrade_rev
]
else:
initial = True
since = options.since
since_option = ['--since=%s' % since, 'origin/master']
repo = git.Repo(repodir)
if repo.bare:
logger.error('Repository %s is bare, not supported' %
repodir)
continue
commits = utils.runcmd(['git', 'log'] + since_option +
['--format=%H %ct', '--reverse'],
repodir,
logger=logger)
commit_list = commits.split('\n')
bitbake_map = {}
def remap_range(start, end, replacewith=None):
if replacewith is None:
replacewith = end
bitbake_commits = utils.runcmd(
['git', 'rev-list',
'%s^..%s^' % (start, end)],
bitbakepath,
logger=logger)
bitbake_commit_list = bitbake_commits.splitlines()
for commit in bitbake_commit_list:
bitbake_map[commit] = replacewith
# Filter out some bad commits
remap_range('fef18b445c0cb6b266cd939b9c78d7cbce38663f',
'39780b1ccbd76579db0fc6fb9369c848a3bafa9d')
remap_range('5796ed550d127853808f38257f8dcc8c1cf59342',
'547128731e62b36d2271c4390b3fee2b16c535dc')
if options.stop_commit and (options.stop_commit not in [
x.split()[0] for x in commit_list
]):
logger.error('Stop commit %s is not in repository %s' %
(options.stop_commit, repodir))
sys.exit(1)
if initial:
logger.debug("Adding initial upgrade history ....")
ct, ctepoch = commit_list.pop(0).split()
ctdate = datetime.fromtimestamp(int(ctepoch))
run_internal(maintplanbranch,
ct,
ctdate,
options,
logger,
bitbake_map,
initial=True)
if layerbranch.vcs_subdir:
layersubdir_start = layerbranch.vcs_subdir
if not layersubdir_start.endswith('/'):
layersubdir_start += '/'
else:
layersubdir_start = ''
logger.debug("Adding upgrade history from %s to %s ..." %
(since, datetime.today().strftime("%Y-%m-%d")))
for item in commit_list:
if item:
ct, ctepoch = item.split()
if ct == options.stop_commit:
logger.debug('Stopping at requested commit %s' %
ct)
break
ctdate = datetime.fromtimestamp(int(ctepoch))
commitobj = repo.commit(ct)
touches_recipe = False
for parent in commitobj.parents:
diff = parent.diff(commitobj)
for diffitem in diff:
if layersubdir_start and not (
diffitem.a_path.startswith(
layersubdir_start) or diffitem.
b_path.startswith(layersubdir_start)):
# Not in this layer, skip it
continue
if options.filter_files and not (
diffitem.a_path.startswith(
options.filter_files)
or diffitem.b_path.startswith(
options.filter_files)):
# Doesn't match path filter
continue
if diffitem.a_path.endswith(
('.bb',
'.inc')) or diffitem.b_path.endswith(
('.bb', '.inc')):
# We need to look at this commit
touches_recipe = True
break
if touches_recipe:
break
if not touches_recipe:
# No recipes in the layer changed in this commit
# NOTE: Whilst it's possible that a change to a class might alter what's
# in the recipe, we can ignore that since we are only concerned with actual
# upgrades which would always require some sort of change to the recipe
# or an include file, so we can safely skip commits that don't do that
logger.debug("Skipping commit %s" % ct)
continue
logger.debug("Analysing commit %s ..." % ct)
run_internal(maintplanbranch, ct, ctdate, options,
logger, bitbake_map)
if not (options.dry_run or options.filter_files):
maintplanbranch.upgrade_rev = ct
maintplanbranch.upgrade_date = ctdate
maintplanbranch.save()
finally:
utils.unlock_file(lockfile)