def __init__(self):
self.groups = {
_group.group_id: _group
for _group in Group.objects.all()
}
self.reporting_periods = {
_period.name: _period
for _period in ReportingPeriod.objects.all()
}
self.parties = {
_party.abbr: _party
for _party in Party.get_main_parties()
}
self.party_types = [_pt.name for _pt in PartyType.objects.all()]
# The number of control measures is relatively low; so to avoid
# repeatedly querying the DB, we can keep all ControlMeasure objects
# in a dictionary with:
# - keys: (group_id, party_type.name, limit_type)
# - values: lists of ControlMeasure objects, ordered by start_date
self.control_measures = {(_group, _party_type, _limit_type.value): []
for _group in self.groups.keys()
for _party_type in self.party_types
for _limit_type in LimitTypes}
for cm in ControlMeasure.objects.select_related(
'group', 'party_type', 'baseline_type').order_by('start_date'):
key = (cm.group.group_id, cm.party_type.name, cm.limit_type)
self.control_measures[key].append(cm)
def get_data(self):
main_parties = Party.get_main_parties()
art7 = Obligation.objects.get(
_obligation_type=ObligationTypes.ART7.value)
self.submissions = Submission.latest_submitted_for_parties(
art7,
self.period,
main_parties,
)
self.group_map = {g.pk: g for g in Group.objects.all()}
import_data = self.aggregates_by_submission_and_substance(
Article7Import)
export_data = self.aggregates_by_submission_and_substance(
Article7Export)
self.impexp_rows = defaultdict(dict)
for party in main_parties:
party_imports = import_data.get(party, {})
party_exports = export_data.get(party, {})
for substance in set(party_imports) | set(party_exports):
self.impexp_rows[party][substance] = {
'import': party_imports.get(substance),
'export': party_exports.get(substance),
}
def handle(self, *args, **options):
stream = logging.StreamHandler()
stream.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
logger.addHandler(stream)
logger.setLevel(logging.INFO)
limits.logger.addHandler(stream)
limits.logger.setLevel(logging.INFO)
if int(options['verbosity']) > 1:
logger.setLevel(logging.DEBUG)
limits.logger.setLevel(logging.DEBUG)
if not ControlMeasure.objects.exists():
logger.error("Control measures not found, aborting.")
return
if options['party']:
parties = (Party.objects.get(abbr=options['party']), )
else:
parties = Party.get_main_parties()
if options['test_file']:
results = self.load_legacy_data(options['test_file'])
self.test_legacy_data(results)
else:
for party in parties:
self.process_party(party, options)
def render_data(self, table_builder):
histories = (PartyHistory.objects.filter(
reporting_period=self.period).filter(is_article5=self.is_article5))
# must filter out parties with submissions in data entry
parties = (Party.get_main_parties().filter(
Q(submissions__reporting_period=self.period)
& Q(submissions__obligation___obligation_type=ObligationTypes.ART7.
value) & ~Q(submissions___current_state='data_entry')).filter(
history__in=histories).distinct())
history_map = {h.party: h for h in histories}
for party in parties:
history = history_map[party]
prodcons_qs = (ProdCons.objects.filter(
reporting_period=self.period).filter(party=party))
prodcons_groups = self.get_prodcons_groups(prodcons_qs)
date_reported, date_revised = get_date_reported(prodcons_qs)
heading = self.render_heading(party, history, date_reported,
date_revised)
table_builder.add_heading(heading)
self.render_groups(
table_builder,
prodcons_groups,
history.population,
show_prod=not party.is_eu,
show_cons=not history.is_eu_member,
)
def _new_eu_member_states_since(period_name):
eu_members_for_period = Party.get_eu_members_at(
self.reporting_periods[period_name]).values_list('abbr',
flat=True)
return [
self.parties[_party] for _party in self.eu_member_states
if _party not in eu_members_for_period
]
def __init__(self):
self.current_period = ReportingPeriod.get_current_period()
self.groups = {
_group.group_id: _group
for _group in Group.objects.all()
}
self.reporting_periods = {
_period.name: _period
for _period in ReportingPeriod.objects.all()
}
self.parties = {
_party.abbr: _party
for _party in Party.get_main_parties()
}
self.party_types = {
entry['party__abbr']: entry['party_type__abbr']
for entry in PartyHistory.objects.filter(
reporting_period=self.current_period, ).values(
'party__abbr', 'party_type__abbr')
}
self.eu_member_states = [
_party.abbr for _party in Party.get_eu_members()
]
self.eu_member_states_1989 = [
_party.abbr for _party in Party.get_eu_members_at(
self.reporting_periods['1989'])
]
self.baseline_types = {
_baseline_type.name: _baseline_type
for _baseline_type in BaselineType.objects.all()
}
def _new_eu_member_states_since(period_name):
eu_members_for_period = Party.get_eu_members_at(
self.reporting_periods[period_name]).values_list('abbr',
flat=True)
return [
self.parties[_party] for _party in self.eu_member_states
if _party not in eu_members_for_period
]
self.new_eu_member_states_since = {
_period: _new_eu_member_states_since(_period)
for _period in ('1989', '2009', '2010')
}
def render_data(self, table_builder):
# global
prodcons_groups = self.get_prodcons_groups(
ProdCons.objects.filter(reporting_period=self.period))
parties = (
Party.get_main_parties()
# must filter out parties with submissions in data entry
.filter(
Q(submissions__reporting_period=self.period)
& Q(submissions__obligation___obligation_type=ObligationTypes.
ART7.value)
& ~Q(submissions___current_state='data_entry')))
stats = self.get_stats(parties)
heading = self.render_heading("All parties", **stats)
table_builder.add_heading(heading)
self.render_groups(table_builder, prodcons_groups, stats['population'])
# regions
for region in self.regions:
prodcons_groups = self.get_prodcons_groups(
ProdCons.objects.filter(reporting_period=self.period).filter(
party__subregion__region=region))
# must filter out parties with submissions in data entry
parties = (Party.get_main_parties(
).filter(subregion__region=region).filter(
Q(submissions__reporting_period=self.period)
& Q(submissions__obligation___obligation_type=ObligationTypes.
ART7.value)
& ~Q(submissions___current_state='data_entry')))
stats = self.get_stats(parties)
heading = self.render_heading(region.name, **stats)
table_builder.add_heading(heading)
self.render_groups(table_builder, prodcons_groups,
stats['population'])
def render_data(self, table_builder):
# global
prodcons_groups = self.get_prodcons_groups(
ProdCons.objects.filter(reporting_period=self.period))
parties = (Party.get_main_parties().filter(
submissions__reporting_period=self.period))
stats = self.get_stats(parties)
heading = self.render_heading("All parties", **stats)
table_builder.add_heading(heading)
self.render_groups(table_builder, prodcons_groups, stats['population'])
# by is_article5
for is_article5 in [True, False]:
histories = (PartyHistory.objects.filter(
reporting_period=self.period).filter(is_article5=is_article5))
# must filter out parties with submissions in data entry
parties = (Party.get_main_parties().filter(
Q(submissions__reporting_period=self.period)
& Q(submissions__obligation___obligation_type=ObligationTypes.
ART7.value)
& ~Q(submissions___current_state='data_entry')).filter(
history__in=histories))
prodcons_groups = self.get_prodcons_groups(
ProdCons.objects.filter(reporting_period=self.period).filter(
party__in=parties))
stats = self.get_stats(parties)
if is_article5:
heading = self.render_heading("Article 5 parties", **stats)
else:
heading = self.render_heading("Non-Article 5 parties", **stats)
table_builder.add_heading(heading)
self.render_groups(table_builder, prodcons_groups,
stats['population'])
def get_parties(request):
if request.user.is_secretariat:
qs = Party.get_main_parties()
else:
related_parties = [request.user.party_id
] if request.user.party_id else []
if request.user.party_group:
related_parties += request.user.party_group.parties.values_list(
'id', flat=True)
qs = Party.objects.filter(pk__in=related_parties)
parties = request.GET.getlist(key='party')
if parties:
qs = qs.filter(pk__in=parties)
return qs.order_by('name')
def handle(self, *args, **options):
stream = logging.StreamHandler()
stream.setFormatter(logging.Formatter(
'%(asctime)s %(levelname)s %(message)s'
))
logger.addHandler(stream)
logger.setLevel(logging.INFO)
if int(options['verbosity']) > 1:
logger.setLevel(logging.DEBUG)
if options['party']:
parties = [Party.objects.get(abbr=options['party'])]
else:
parties = Party.get_main_parties()
if options['period']:
periods = [ReportingPeriod.objects.get(name=options['period'])]
else:
# Only use reporting periods for which there are PartyHistory
# entries
existing_histories = PartyHistory.objects.all().values_list(
'reporting_period__name', flat=True
)
periods = list(ReportingPeriod.objects.filter(
name__in=existing_histories)
)
if not options['confirm']:
logger.info(
f"Run with --confirm to update actual baseline/limit data for "
f"{10 * len(parties) * len(periods)} ActualBaselineAndLimit"
f"objects."
)
for period in periods:
for party in parties:
if options['confirm']:
logger.info(f"Updating data for {period} - {party}")
else:
logger.debug(f"Would update data for {period} - {party}")
if options['confirm']:
values = ActualBaselineAndLimit.populate_actual_data(
party, period
)
logger.debug(values)
def admin_diff(request, context):
parties = Party.get_main_parties()
if request.POST['party'] != '*':
parties = parties.filter(pk=request.POST['party'])
reporting_periods = ReportingPeriod.objects.all()
if request.POST['reporting_period'] != '*':
reporting_periods = reporting_periods.filter(
pk=request.POST['reporting_period'])
groups = Group.objects.all()
if request.POST['group'] != '*':
groups = groups.filter(pk=request.POST['group'])
diff = limits_diff(parties, reporting_periods, groups)
for record in diff['missing']:
record['checkbox_value'] = json.dumps({
'party_id':
record['party'].id,
'reporting_period_id':
record['reporting_period'].id,
'limit_type':
record['limit_type'].value,
'group_id':
record['group'].id,
'limit':
str(record['limit']),
})
for record in diff['different']:
record['checkbox_value'] = json.dumps({
'pk':
record['row'].pk,
'limit':
str(record['new_value']),
})
for record in diff['obsolete']:
record['checkbox_value'] = json.dumps({
'pk': record['row'].pk,
})
context['diff'] = diff
def admin_view(request, context):
if request.POST:
step = request.POST['step']
context['step'] = step
if step == 'diff':
admin_diff(request, context)
elif step == 'apply':
admin_apply(request, context)
else:
raise RuntimeError(f"Unexpected step {step!r}")
context['parties'] = Party.get_main_parties()
context['groups'] = Group.objects.all()
return TemplateResponse(request,
'admin/ozone_tools/generate_baselines.html',
context)
def admin_view(request, context):
if request.POST:
step = request.POST['step']
context['step'] = step
if step == 'diff':
admin_diff(request, context)
elif step == 'apply':
admin_apply(request, context)
else:
raise RuntimeError(f"Unexpected step {step!r}")
context['parties'] = Party.get_main_parties()
context['reporting_periods'] = ReportingPeriod.objects.order_by(
'-start_date').all()
context['groups'] = Group.objects.all()
return TemplateResponse(request, 'admin/ozone_tools/generate_limits.html',
context)
def admin_diff(request, context):
parties = Party.get_main_parties()
if request.POST['party'] != '*':
parties = parties.filter(pk=request.POST['party'])
groups = Group.objects.all()
if request.POST['group'] != '*':
groups = groups.filter(pk=request.POST['group'])
diff = baselines_diff(parties, groups)
for record in diff['missing']:
record['checkbox_value'] = json.dumps({
'party_id':
record['party'].id,
'baseline_type_id':
record['baseline_type'].id,
'group_id':
record['group'].id,
'baseline':
str(record['baseline']),
})
for record in diff['different']:
record['checkbox_value'] = json.dumps({
'pk':
record['row'].pk,
'baseline':
str(record['new_value']),
})
for record in diff['obsolete']:
record['checkbox_value'] = json.dumps({
'pk': record['row'].pk,
})
context['diff'] = diff
def handle(self, *args, **options):
stream = logging.StreamHandler()
stream.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
logger.addHandler(stream)
logger.setLevel(logging.INFO)
baselines.logger.addHandler(stream)
baselines.logger.setLevel(logging.INFO)
if int(options['verbosity']) > 1:
logger.setLevel(logging.DEBUG)
baselines.logger.setLevel(logging.DEBUG)
if options['party']:
parties = (Party.objects.get(abbr=options['party']), )
else:
parties = Party.get_main_parties()
if options['test_file']:
results = self.load_legacy_data(options['test_file'])
self.test_legacy_data(results)
else:
for party in parties:
self.process_party(party, options)
def handle(self, *args, **options):
if not ControlMeasure.objects.exists():
print("Control measures not found. Please, "
"import control measure fixtures.")
return
data = []
idx = 1
for party_history in PartyHistory.objects.all():
party = party_history.party
party_type = party_history.party_type
period = party_history.reporting_period
if period.is_control_period:
# No limits for control periods
continue
print('Processing country {} and period {}'.format(
party.name, period.name))
for group in Group.objects.all():
cm_queryset = ControlMeasure.objects.filter(
group=group,
party_type=party_type,
start_date__lte=period.end_date).filter(
Q(end_date__gte=period.start_date)
| Q(end_date__isnull=True)).order_by('start_date')
for limit_type in LimitTypes:
if limit_type.value in [
LimitTypes.BDN.value, LimitTypes.PRODUCTION.value
] and party.abbr == 'EU':
# No BDN or Prod limits for EU/ECE(European Union)
continue
if limit_type.value in [
LimitTypes.CONSUMPTION.value
] and party in Party.get_eu_members_at(period):
# No consumption baseline for EU member states
continue
cm_queryset_by_limit_type = cm_queryset.filter(
limit_type=limit_type.value)
length = cm_queryset_by_limit_type.count()
if length == 0:
continue
elif length == 1:
cm = cm_queryset_by_limit_type.first()
if group.group_id == 'CII' or group.group_id == 'CIII':
baseline = 0
else:
baseline = Baseline.objects.filter(
party=party,
group=group,
baseline_type=cm.baseline_type).first()
if not baseline or baseline.baseline is None:
continue
else:
baseline = baseline.baseline
# TODO: rounding
limit = cm.allowed * baseline
data.append(
self.get_entry(idx, party, period, group,
limit_type.value, limit))
idx += 1
elif length == 2:
# This happens for BDN limits, A/I and E/I, Non-A5 parties
if group.group_id == 'CII' or group.group_id == 'CIII':
baseline1 = Decimal('0')
baseline2 = Decimal('0')
else:
cm1 = cm_queryset_by_limit_type[0]
cm2 = cm_queryset_by_limit_type[1]
baseline1 = Baseline.objects.filter(
party=party,
group=group,
baseline_type=cm1.baseline_type).first()
baseline2 = Baseline.objects.filter(
party=party,
group=group,
baseline_type=cm2.baseline_type).first()
if (not baseline1 or baseline1.baseline is None
or not baseline2
or baseline2.baseline is None):
continue
else:
baseline1 = baseline1.baseline
baseline2 = baseline2.baseline
days1 = (cm1.end_date - period.start_date).days + 1
days2 = (period.end_date - cm2.start_date).days + 1
limit = (
(100 * cm1.allowed * days1 * baseline1) / 100 +
(100 * cm2.allowed * days2 * baseline2) / 100) / (
(period.end_date - period.start_date).days + 1)
data.append(
self.get_entry(idx, party, period, group,
limit_type.value, limit))
idx += 1
filename = os.path.join(self.OUTPUT_DIR, 'limits.json')
with open(filename, 'w', encoding="utf-8") as outfile:
json.dump(data,
outfile,
indent=2,
ensure_ascii=False,
sort_keys=True,
cls=DjangoJSONEncoder)
print('Done with %s' % filename)
def _party_in_eu(self, party, period):
return party in Party.get_eu_members_at(period)
def _prod_cons_f(self, party, _group, _periods, prod_cons,
base_party_type): # noqa: C901
"""
HFC baseline actually depends on the party group:
NA5 Group 1: Average HFC for 2011-2013 + 15% of HCFC baseline
NA5 Group 2: Average HFC for 2011-2013 + 25% of HCFC baseline
A5 Group 1: Average HFC for 2020–2022 + 65% of HCFC baseline
A5 Group 2: Average HFC for 2024–2026 + 65% of HCFC baseline
"""
party_type = self.party_types.get(party.abbr, None)
if party_type is None:
# can happen for Palestine or new states
logger.warning(
f"No party history for {party.abbr}/{self.current_period.name}"
)
return None
if not party_type.startswith(base_party_type):
return None
base_periods = {
'NA5G1': ('2011', '2012', '2013'),
'NA5G2': ('2011', '2012', '2013'),
'A5G1': ('2020', '2021', '2022'),
'A5G2': ('2024', '2025', '2026'),
}.get(party_type)
hfc_group = self.groups.get('F')
total_amount = Decimal('0.0')
for period in base_periods:
prodcons = self._get_prodcons(party, hfc_group, period)
if not prodcons:
# If data not reported, no baseline is computed
return None
if prod_cons == 'PROD':
total_amount += prodcons.calculated_production
elif prod_cons == 'CONS':
# Check EU membership
if party in Party.get_eu_members_at(
self.reporting_periods[period]):
return None
total_amount += (prodcons.calculated_consumption
if prodcons.calculated_consumption else
Decimal('0'))
if total_amount < 0:
total_amount = Decimal('0')
hcfc_group = self.groups.get('CI')
hcfc_baseline_type = {
('PROD', 'NA5G1'): 'NA5ProdGWP',
('PROD', 'NA5G2'): 'NA5ProdGWP',
('PROD', 'A5G1'): 'A5ProdGWP',
('PROD', 'A5G2'): 'A5ProdGWP',
('CONS', 'NA5G1'): 'NA5ConsGWP',
('CONS', 'NA5G2'): 'NA5ConsGWP',
('CONS', 'A5G1'): 'A5ConsGWP',
('CONS', 'A5G2'): 'A5ConsGWP',
}.get((prod_cons, party_type))
hcfc_baseline = self.get_baseline(hcfc_baseline_type, hcfc_group,
party)
if hcfc_baseline is None:
return None
rounded_average = round_decimal_half_up(
sum((
total_amount / len(base_periods),
self._get_hcfc_percentage(party) * hcfc_baseline,
)), 0)
return rounded_average if rounded_average > 0 else Decimal('0')