def rrs_import(args):
utils.setup_django()
import settings
from django.db import transaction
from rrs.models import RecipeUpstreamHistory, RecipeUpstream
from layerindex.models import Recipe
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
if not core_layer:
logger.error('Unable to find core layer %s' % settings.CORE_LAYER_NAME)
return 1
core_layerbranch = core_layer.get_layerbranch('master')
if not core_layerbranch:
logger.error('Unable to find branch master of layer %s' % core_layerbranch.name)
return 1
layerbranch = core_layerbranch
try:
with transaction.atomic():
with open(args.infile, 'r') as f:
data = json.load(f)
for item, itemdata in data.items():
if item == 'recipeupstreamhistory':
for histdata in itemdata:
ruh = RecipeUpstreamHistory()
ruh.start_date = histdata['start_date']
ruh.end_date = histdata['end_date']
ruh.layerbranch = layerbranch
ruh.save()
for upstreamdata in histdata['upstreams']:
ru = RecipeUpstream()
ru.history = ruh
pn = upstreamdata['recipe']
recipe = Recipe.objects.filter(layerbranch=layerbranch, pn=pn).first()
if not recipe:
logger.warning('Could not find recipe %s in layerbranch %s' % (pn, layerbranch))
continue
ru.recipe = recipe
ru.version = upstreamdata['version']
ru.type = upstreamdata['type']
ru.status = upstreamdata['status']
ru.no_update_reason = upstreamdata['no_update_reason']
ru.date = upstreamdata['date']
ru.save()
if args.dry_run:
raise DryRunRollbackException
except DryRunRollbackException:
pass
return 0
settings,
logger,
recipe_files=recipe_files)
try:
if not recipes:
continue
utils.setup_core_layer_sys_path(
settings, layerbranch.branch.name)
for recipe_data in recipes:
set_regexes(recipe_data)
history = RecipeUpstreamHistory(
layerbranch=layerbranch,
start_date=datetime.now())
result = []
for recipe_data in recipes:
try:
get_upstream_info(layerbranch, recipe_data,
result)
except:
import traceback
traceback.print_exc()
history.end_date = datetime.now()
history.save()
for res in result:
def main():
parser = optparse.OptionParser(usage="""
%prog [options]""")
parser.add_option(
"-p",
"--plan",
help=
"Specify maintenance plan to operate on (default is all plans that have updates enabled)",
action="store",
dest="plan",
default=None)
parser.add_option("-s",
"--subject",
action="store",
dest="subject",
help='Override email subject')
parser.add_option("-f",
"--from",
action="store",
dest="_from",
help='Override sender address')
parser.add_option("-t",
"--to",
action="store",
dest="to",
help='Override recipient address')
parser.add_option("-d",
"--debug",
help="Enable debug output",
action="store_const",
const=logging.DEBUG,
dest="loglevel",
default=logging.INFO)
parser.add_option("-q",
"--quiet",
help="Hide all output except error messages",
action="store_const",
const=logging.ERROR,
dest="loglevel")
options, args = parser.parse_args(sys.argv)
logger.setLevel(options.loglevel)
# get recipes for send email
if options.plan:
maintplans = MaintenancePlan.objects.filter(id=int(options.plan))
if not maintplans.exists():
logger.error('No maintenance plan with ID %s found' % options.plan)
sys.exit(1)
else:
maintplans = MaintenancePlan.objects.filter(email_enabled=True)
if not maintplans.exists():
logger.error('No maintenance plans with email enabled were found')
sys.exit(1)
for maintplan in maintplans:
recipes = {}
for item in maintplan.maintenanceplanlayerbranch_set.all():
layerbranch = item.layerbranch
recipe_upstream_history = RecipeUpstreamHistory.get_last(
layerbranch)
if recipe_upstream_history is None:
logger.warn(
'I don\'t have Upstream information yet, run update.py script'
)
sys.exit(1)
recipe_maintainer_history = RecipeMaintainerHistory.get_last(
layerbranch)
if recipe_maintainer_history is None:
logger.warn('I don\'t have Maintainership information yet,' +
' run rrs_maintainer_history.py script')
sys.exit(1)
for recipe in layerbranch.recipe_set.all():
recipe_upstream_query = RecipeUpstream.objects.filter(
recipe=recipe, history=recipe_upstream_history)
if recipe_upstream_query and recipe_upstream_query[
0].status == 'N':
recipes[recipe] = {}
recipe_maintainer = RecipeMaintainer.objects.filter(
recipe=recipe, history=recipe_maintainer_history)[0]
recipes[recipe]['maintainer'] = recipe_maintainer
recipes[recipe]['upstream'] = recipe_upstream_query[0]
send_email(maintplan, recipes, options)
def get_context_data(self, **kwargs):
context = super(RecipeDetailView, self).get_context_data(**kwargs)
recipe = self.get_object()
if not recipe:
raise django.http.Http404
maintplan = get_object_or_404(MaintenancePlan,
name=self.maintplan_name)
context['maintplan_name'] = maintplan.name
context['maintplan'] = maintplan
release = Release.get_current(maintplan)
context['release_name'] = release.name
milestone = Milestone.get_current(release)
context['milestone_name'] = milestone.name
context['upstream_status'] = ''
context['upstream_version'] = ''
context['upstream_no_update_reason'] = ''
recipe_upstream_history = RecipeUpstreamHistory.get_last_by_date_range(
recipe.layerbranch, milestone.start_date, milestone.end_date)
if recipe_upstream_history:
recipe_upstream = RecipeUpstream.get_by_recipe_and_history(
recipe, recipe_upstream_history)
if recipe_upstream:
if recipe_upstream.status == 'N' and recipe_upstream.no_update_reason:
recipe_upstream.status = 'C'
elif recipe_upstream.status == 'D':
recipe_upstream.status = 'U'
context['upstream_status'] = \
RecipeUpstream.RECIPE_UPSTREAM_STATUS_CHOICES_DICT[recipe_upstream.status]
context['upstream_version'] = recipe_upstream.version
context[
'upstream_no_update_reason'] = recipe_upstream.no_update_reason
self.recipe_maintainer_history = RecipeMaintainerHistory.get_last(
recipe.layerbranch)
recipe_maintainer = RecipeMaintainer.objects.filter(
recipe=recipe, history=self.recipe_maintainer_history)
if recipe_maintainer:
maintainer = recipe_maintainer[0].maintainer
context['maintainer_name'] = maintainer.name
else:
context['maintainer_name'] = 'No maintainer'
context['recipe_upgrade_details'] = []
for ru in RecipeUpgrade.objects.filter(
recipe=recipe).order_by('-commit_date'):
context['recipe_upgrade_details'].append(
_get_recipe_upgrade_detail(maintplan, ru))
context['recipe_upgrade_detail_count'] = len(
context['recipe_upgrade_details'])
context['recipe_layer_branch_url'] = _get_layer_branch_url(
recipe.layerbranch.branch.name, recipe.layerbranch.layer.name)
context['recipe_provides'] = []
for p in recipe.provides.split():
context['recipe_provides'].append(p)
context['recipe_depends'] = StaticBuildDep.objects.filter(
recipes__id=recipe.id).values_list('name', flat=True)
context['recipe_distros'] = RecipeDistro.get_distros_by_recipe(recipe)
return context
def _get_recipe_list(milestone):
recipe_list = []
recipes_ids = []
recipe_upstream_dict_all = {}
recipe_last_updated_dict_all = {}
maintainers_dict_all = {}
current_date = date.today()
for maintplanlayer in milestone.release.plan.maintenanceplanlayerbranch_set.all(
):
layerbranch = maintplanlayer.layerbranch
recipe_maintainer_history = Raw.get_remahi_by_end_date(
layerbranch.id, milestone.end_date)
recipe_upstream_history = RecipeUpstreamHistory.get_last_by_date_range(
layerbranch, milestone.start_date, milestone.end_date)
recipes = Raw.get_reupg_by_date(layerbranch.id, milestone.end_date)
for i, re in enumerate(recipes):
if 'pv' in re:
recipes[i]['version'] = re['pv']
recipes_ids.append(re['id'])
if recipes:
recipe_last_updated = Raw.get_reup_by_last_updated(
layerbranch.id, milestone.end_date)
for rela in recipe_last_updated:
recipe_last_updated_dict_all[rela['recipe_id']] = rela
if recipe_upstream_history:
recipe_upstream_all = Raw.get_reup_by_recipes_and_date(
recipes_ids, recipe_upstream_history.id)
for reup in recipe_upstream_all:
recipe_upstream_dict_all[reup['recipe_id']] = reup
if recipe_maintainer_history:
maintainers_all = Raw.get_ma_by_recipes_and_date(
recipes_ids, recipe_maintainer_history[0])
for ma in maintainers_all:
maintainers_dict_all[ma['recipe_id']] = ma['name']
for recipe in recipes:
upstream_version = ''
upstream_status = ''
no_update_reason = ''
outdated = ''
if recipe_upstream_history:
recipe_upstream = recipe_upstream_dict_all.get(recipe['id'])
if not recipe_upstream:
recipe_add = Recipe.objects.filter(id=recipe['id'])[0]
recipe_upstream_add = RecipeUpstream()
recipe_upstream_add.history = recipe_upstream_history
recipe_upstream_add.recipe = recipe_add
recipe_upstream_add.version = ''
recipe_upstream_add.type = 'M' # Manual
recipe_upstream_add.status = 'U' # Unknown
recipe_upstream_add.no_update_reason = ''
recipe_upstream_add.date = recipe_upstream_history.end_date
recipe_upstream_add.save()
recipe_upstream = {
'version': '',
'status': 'U',
'type': 'M',
'no_update_reason': ''
}
if recipe_upstream['status'] == 'N' and recipe_upstream[
'no_update_reason']:
recipe_upstream['status'] = 'C'
upstream_status = \
RecipeUpstream.RECIPE_UPSTREAM_STATUS_CHOICES_DICT[
recipe_upstream['status']]
if upstream_status == 'Downgrade':
upstream_status = 'Unknown' # Downgrade is displayed as Unknown
upstream_version = recipe_upstream['version']
no_update_reason = recipe_upstream['no_update_reason']
#Get how long the recipe hasn't been updated
recipe_last_updated = \
recipe_last_updated_dict_all.get(recipe['id'])
if recipe_last_updated:
recipe_date = recipe_last_updated['date']
outdated = recipe_date.date().isoformat()
else:
outdated = ""
maintainer_name = maintainers_dict_all.get(recipe['id'], '')
recipe_list_item = RecipeList(recipe['id'], recipe['pn'],
recipe['summary'])
recipe_list_item.version = recipe['version']
recipe_list_item.upstream_status = upstream_status
recipe_list_item.upstream_version = upstream_version
recipe_list_item.outdated = outdated
patches = Patch.objects.filter(recipe__id=recipe['id'])
recipe_list_item.patches_total = patches.count()
recipe_list_item.patches_pending = patches.filter(status='P').count()
recipe_list_item.maintainer_name = maintainer_name
recipe_list_item.no_update_reason = no_update_reason
recipe_list.append(recipe_list_item)
return recipe_list
def _get_milestone_statistics(milestone, maintainer_name=None):
milestone_statistics = {}
milestone_statistics['all'] = 0
milestone_statistics['up_to_date'] = 0
milestone_statistics['not_updated'] = 0
milestone_statistics['cant_be_updated'] = 0
milestone_statistics['unknown'] = 0
if maintainer_name is None:
milestone_statistics['all_upgraded'] = 0
milestone_statistics['all_not_upgraded'] = 0
for maintplanlayer in milestone.release.plan.maintenanceplanlayerbranch_set.all(
):
layerbranch = maintplanlayer.layerbranch
recipe_upstream_history = RecipeUpstreamHistory.get_last_by_date_range(
layerbranch, milestone.start_date, milestone.end_date)
recipe_upstream_history_first = \
RecipeUpstreamHistory.get_first_by_date_range(
layerbranch,
milestone.start_date,
milestone.end_date,
)
if maintainer_name is None:
t_updated, t_not_updated, t_cant, t_unknown = \
Raw.get_reup_statistics(milestone.release.plan, milestone.end_date, recipe_upstream_history)
milestone_statistics['all'] += \
t_updated + t_not_updated + t_cant + t_unknown
milestone_statistics['up_to_date'] = +t_updated
milestone_statistics['not_updated'] = +t_not_updated
milestone_statistics['cant_be_updated'] += t_cant
milestone_statistics['unknown'] += t_unknown
if recipe_upstream_history_first:
recipes_not_upgraded = \
Raw.get_reup_by_date(recipe_upstream_history_first.id)
if recipes_not_upgraded:
recipes_upgraded = \
Raw.get_reupg_by_dates_and_recipes(
milestone.start_date, milestone.end_date, recipes_not_upgraded)
milestone_statistics['all_upgraded'] += len(
recipes_upgraded)
milestone_statistics['all_not_upgraded'] += len(
recipes_not_upgraded)
else:
recipe_maintainer_history = Raw.get_remahi_by_end_date(
layerbranch.id, milestone.end_date)
recipe_maintainer_all = Raw.get_re_by_mantainer_and_date(
maintainer_name, recipe_maintainer_history[0])
milestone_statistics['all'] += len(recipe_maintainer_all)
if recipe_upstream_history:
recipe_upstream_all = Raw.get_reup_by_recipes_and_date(
recipe_maintainer_all, recipe_upstream_history.id)
else:
recipe_upstream_all = Raw.get_reup_by_recipes_and_date(
recipe_maintainer_all)
for ru in recipe_upstream_all:
if ru['status'] == 'Y':
milestone_statistics['up_to_date'] += 1
elif ru['status'] == 'N':
if ru['no_update_reason'] == '':
milestone_statistics['not_updated'] += 1
else:
milestone_statistics['cant_be_updated'] += 1
else:
milestone_statistics['unknown'] += 1
milestone_statistics['percentage'] = '0'
if maintainer_name is None:
if milestone_statistics['all'] > 0:
milestone_statistics['percentage_up_to_date'] = "%.0f" % \
(float(milestone_statistics['up_to_date']) * 100.0 \
/float(milestone_statistics['all']))
milestone_statistics['percentage_not_updated'] = "%.0f" % \
(float(milestone_statistics['not_updated']) * 100.0 \
/float(milestone_statistics['all']))
milestone_statistics['percentage_cant_be_updated'] = "%.0f" % \
(float(milestone_statistics['cant_be_updated']) * 100.0 \
/float(milestone_statistics['all']))
milestone_statistics['percentage_unknown'] = "%.0f" % \
(float(milestone_statistics['unknown']) * 100.0
/float(milestone_statistics['all']))
if milestone_statistics['all_not_upgraded'] > 0:
milestone_statistics['percentage'] = "%.0f" % \
((float(milestone_statistics['all_upgraded']) * 100.0)
/float(milestone_statistics['all_not_upgraded']))
else:
milestone_statistics['percentage_up_to_date'] = "0"
milestone_statistics['percentage_not_updated'] = "0"
milestone_statistics['percentage_cant_be_updated'] = "0"
milestone_statistics['percentage_unknown'] = "0"
else:
if milestone_statistics['all'] > 0:
milestone_statistics['percentage'] = "%.0f" % \
((float(milestone_statistics['up_to_date']) /
float(milestone_statistics['all'])) * 100)
return milestone_statistics
def get_context_data(self, **kwargs):
context = super(RecipeDetailView, self).get_context_data(**kwargs)
recipesymbol = self.get_object()
if not recipesymbol:
raise django.http.Http404
recipe = recipesymbol.layerbranch.recipe_set.filter(
pn=recipesymbol.pn, layerbranch=recipesymbol.layerbranch).last()
context['recipe'] = recipe
maintplan = get_object_or_404(MaintenancePlan,
name=self.maintplan_name)
context['maintplan_name'] = maintplan.name
context['maintplan'] = maintplan
release = Release.get_current(maintplan)
context['release_name'] = release.name
milestone = Milestone.get_current(release)
context['milestone_name'] = milestone.name
context['upstream_status'] = ''
context['upstream_version'] = ''
context['upstream_no_update_reason'] = ''
recipe_upstream_history = RecipeUpstreamHistory.get_last_by_date_range(
recipesymbol.layerbranch, milestone.start_date, milestone.end_date)
if recipe_upstream_history:
recipe_upstream = RecipeUpstream.get_by_recipe_and_history(
recipesymbol, recipe_upstream_history)
if recipe_upstream:
if recipe_upstream.status == 'N' and recipe_upstream.no_update_reason:
recipe_upstream.status = 'C'
elif recipe_upstream.status == 'D':
recipe_upstream.status = 'U'
context['upstream_status'] = \
RecipeUpstream.RECIPE_UPSTREAM_STATUS_CHOICES_DICT[recipe_upstream.status]
context['upstream_version'] = recipe_upstream.version
context[
'upstream_no_update_reason'] = recipe_upstream.no_update_reason
self.recipe_maintainer_history = RecipeMaintainerHistory.get_last(
recipesymbol.layerbranch)
recipe_maintainer = RecipeMaintainer.objects.filter(
recipesymbol=recipesymbol, history=self.recipe_maintainer_history)
if recipe_maintainer:
maintainer = recipe_maintainer[0].maintainer
context['maintainer_name'] = maintainer.name
else:
context['maintainer_name'] = 'No maintainer'
details = []
multigroup = False
lastgroup = '' # can't use None here
for ru in RecipeUpgrade.objects.filter(
recipesymbol=recipesymbol).exclude(upgrade_type='M').order_by(
'group', '-commit_date', '-id'):
details.append(_get_recipe_upgrade_detail(maintplan, ru))
if not multigroup:
if lastgroup == '':
lastgroup = ru.group
elif ru.group != lastgroup:
multigroup = True
details.sort(key=lambda s: RecipeUpgradeGroupSortItem(s.group),
reverse=True)
context['multigroup'] = multigroup
context['recipe_upgrade_details'] = details
context['recipe_upgrade_detail_count'] = len(details)
if not recipe:
ru = RecipeUpgrade.objects.filter(
recipesymbol=recipesymbol).order_by('-commit_date',
'-id').first()
if ru:
context['last_filepath'] = ru.filepath
context['recipe_layer_branch_url'] = _get_layer_branch_url(
recipesymbol.layerbranch.branch.name,
recipesymbol.layerbranch.layer.name)
context['recipe_provides'] = []
if recipe:
for p in recipe.provides.split():
context['recipe_provides'].append(p)
context['recipe_depends'] = StaticBuildDep.objects.filter(
recipes__id=recipe.id).values_list('name', flat=True)
context['recipe_distros'] = RecipeDistro.get_distros_by_recipe(
recipe)
else:
context['recipe_depends'] = []
context['recipe_distros'] = []
context['otherbranch_recipes'] = Recipe.objects.filter(
layerbranch__layer=recipesymbol.layerbranch.layer,
layerbranch__branch__comparison=False,
pn=recipesymbol.pn).order_by('layerbranch__branch__sort_priority')
return context