def setup_data(self):
t = sql.MSFD9Descriptor
muids = {m.id: m for m in self.muids}
count, res = db.get_all_records(
t,
t.MarineUnitID.in_(muids.keys()),
)
by_muid = defaultdict(list)
descriptor = get_descriptor(self.descriptor)
ok_ges_ids = descriptor.all_ids()
for desc_item in res:
ges_id = criteria_from_gescomponent(desc_item.ReportingFeature)
if ges_id not in ok_ges_ids:
continue
item = A9AlternateItem(desc_item)
by_muid[desc_item.MarineUnitID].append(item)
self.rows = {}
for muid, cols in by_muid.items():
rows = []
if not cols:
continue
for name in cols[0].keys():
values = [c[name] for c in cols]
row = Row(name, values)
rows.append(row)
self.rows[muids[muid]] = rows
def setup_data(self):
descriptor = get_descriptor(self.descriptor)
ok_ges_ids = set(descriptor.all_ids())
t = sql.MSFD10Target
# muids = [x.id for x in self.muids]
muids = {m.id: m for m in self.muids}
count, res = db.get_all_records(
t,
t.MarineUnitID.in_(muids.keys()),
t.Topic == 'EnvironmentalTarget',
)
by_muid = defaultdict(list)
for target_item in res:
item = A10AlternateItem(target_item, ok_ges_ids)
if item.needed:
by_muid[target_item.MarineUnitID].append(item)
self.rows = {}
for muid, cols in by_muid.items():
rows = []
if not cols:
continue
for name in cols[0].keys():
values = [c[name] for c in cols]
row = Row(name, values)
rows.append(row)
self.rows[muids[muid]] = rows
def setup_data(self):
# setup data to be available at 'features_pressures_data' attr
self.get_features_pressures_data()
data = self.features_pressures_data
out = []
general_pressures = set([
'PresAll',
]) # 'Unknown'
for theme, features_for_theme in self.features_needed:
theme_data = []
theme_general_pressures = set(
[x for x in features_for_theme if x.endswith('All')])
for feature in features_for_theme:
if feature.endswith('All'):
continue
feature_data = []
for country_id, country_name in self.available_countries:
# if pressure is ending with 'All' it applies to all
# features in the current theme
descriptors_rep = []
for x in data:
descr = x[1]
try:
descr_obj = get_descriptor(descr)
descr = descr_obj.template_vars['title']
descr_name = descr_obj.name
except:
descr_name = descr
ccode = x[0]
if ccode != country_id:
continue
feats = set(x[2].split(','))
if (feature in feats
or general_pressures.intersection(feats) or
theme_general_pressures.intersection(feats)):
descriptors_rep.append((descr_name, descr))
descriptors_rep = set(descriptors_rep)
descriptors_rep = [ItemLabel(*x) for x in descriptors_rep]
feature_data.append(ItemListOverview(descriptors_rep))
theme_data.append(
(self.get_feature_short_name(feature), feature_data))
out.append((theme, theme_data))
return out
def all_descriptor_ids(self):
descr_class = get_descriptor(self.descriptor)
all_ids = list(descr_class.all_ids())
if self.descriptor.startswith('D1.'):
all_ids.append('D1')
all_ids = set(all_ids)
return all_ids
def get_data_from_view_Art9(self):
t = sql2018.t_V_ART9_GES_2018
descriptor = get_descriptor(self.descriptor)
all_ids = list(descriptor.all_ids())
if self.descriptor.startswith('D1.'):
all_ids.append('D1')
conditions = [
t.c.CountryCode == self.country_code,
t.c.GESComponent.in_(all_ids)
]
if self.country_code != 'DK':
conditions.insert(
1,
or_(t.c.Region == self.country_region_code,
t.c.Region.is_(None)))
else:
# Handle the case of Denmark that have submitted a lot of
# information under the DK-TOTAL MRU, which doesn't have a region
# attached.
conditions.insert(
1,
or_(t.c.Region == 'NotReported',
t.c.Region == self.country_region_code,
t.c.Region.is_(None)))
count, q = db.get_all_records_ordered(t, ('GESComponent', ),
*conditions)
ok_features = set([f.name for f in get_features(self.descriptor)])
out = []
# There are cases when justification for delay is reported
# for a ges component. In these cases region, mru, features and
# other fields are empty. Justification for delay should be showed
# for all regions, mrus
for row in q:
if not row.Features:
out.append(row)
continue
if not self.descriptor.startswith('D1.'):
out.append(row)
continue
feats = set(row.Features.split(','))
if feats.intersection(ok_features):
out.append(row)
return out
def insert_missing_criterions(data, descriptor):
""" For Art9 we want to show a row for all possible GESComponents,
regardless if the MS has reported on that or not
:param data: a map of <muid>: list of rows
:param descriptor: a Descriptor instance
This function will change in place the provided data
"""
criterions = []
# need to add D1 descriptor to the criterion lists too, CY reported data
# on the generic D1 descriptor
if 'D1.' in descriptor.id:
criterions.append(get_descriptor('D1'))
criterions.append(descriptor)
criterions.extend(descriptor.sorted_criterions())
for muidlist, dataset in data.items():
# build a map of existing criterions, so that we detect missing ones
# this proxy object will serve as template for new cloned columns,
# to be able to show an empty column
tplobj = None
# colmap = {}
colmap = defaultdict(list)
new = []
for col in dataset:
# rewrite the GESComponent feature. TODO: move this functionality
# to the Proxy2018 and XML file, with a getter
# if col.GESComponent in colmap:
# continue
if col.GESComponent.is_descriptor():
colmap[col.GESComponent].append(col)
else:
col.GESComponent = descriptor[col.GESComponent.id]
colmap[col.GESComponent].append(col)
if tplobj is None:
tplobj = col
for c in criterions:
if c in colmap:
col = colmap[c]
else:
col = [tplobj.clone(GESComponent=c)]
new.extend(col)
data[muidlist] = new
def descriptors(self):
descriptors = get_all_descriptors()
descriptors.pop(0) # remove D1 general descriptor
res = []
for desc in descriptors:
desc_obj = get_descriptor(desc[0])
res.append((desc_obj.template_vars['title'], desc_obj.title))
return res
def filtered_ges_components(self, seed):
""" Returns a list of valid ges criterion indicator targets
Can be something like "1.6.2-indicator 5.2B" or "3.1" or "D1"
"""
descriptor = get_descriptor(self.descriptor)
country_criterions = country_ges_components(self.country_code)
res = set([self.descriptor])
for d_id in descriptor.all_ids():
if d_id in country_criterions:
res.add(d_id)
for crit in set(country_criterions + seed):
crit_id = crit.split('-', 1)[0]
if crit_id in descriptor.all_ids():
res.add(crit)
return sorted_by_criterion(res)
def _art_4_ids(self, descriptor, **kwargs):
""" Return all descriptors """
descriptors = get_all_descriptors()
res = []
for descriptor in descriptors:
descr_id = descriptor[0]
if descr_id == 'D1':
continue
descr_title = descriptor[1]
descriptor_obj = get_descriptor(descr_id)
alternate_id = descriptor_obj.template_vars['title']
descr_opt = DescriptorOption(id=alternate_id,
title=descr_title,
is_primary=lambda _: True)
res.append(descr_opt)
return res
def get_criterias_list(self, descriptor):
""" Get the list of criterias for the specified descriptor
:param descriptor: 'D5'
:return: (('D5', 'Eutrophication'),
('5.1.1', 'D5C1'),
('5.2.1', 'D5C2'), ... )
# TODO: the results here need to be augumented by L_GESComponents
"""
result = [(descriptor, self.descriptor_label)]
criterions = get_descriptor(descriptor).criterions
for crit in criterions:
for alt in crit.alternatives:
title = '{} ({}) {}'.format(crit._id or '', alt[0], alt[1])
indicator = alt[0]
result.append((indicator, title))
return result
def get_data_from_view_Art8(self):
sess = db.session()
t = sql2018.t_V_ART8_GES_2018
descr_class = get_descriptor(self.descriptor)
all_ids = list(descr_class.all_ids())
if self.descriptor.startswith('D1.'):
all_ids.append('D1')
# muids = [x.id for x in self.muids]
conditions = [
t.c.CountryCode == self.country_code,
# t.c.Region == self.country_region_code,
# t.c.MarineReportingUnit.in_(muids), #
t.c.GESComponent.in_(all_ids)
]
# Handle the case of Romania that submitted duplicate data,
# where Element is empty, but Criteria has data
if self.country_code != 'RO':
conditions.append(
or_(t.c.Element.isnot(None), t.c.Criteria.isnot(None)))
else:
conditions.append(t.c.Element.isnot(None))
if self.country_code != 'DK':
conditions.insert(1, t.c.Region == self.country_region_code)
else:
# Handle the case of Denmark that have submitted a lot of
# information under the DK-TOTAL MRU, which doesn't have a region
# attached.
conditions.insert(
1,
or_(t.c.Region == 'NotReported',
t.c.Region == self.country_region_code))
orderby = [
getattr(t.c, x) for x in self._get_order_cols_Art8(self.descriptor)
]
# groupby IndicatorCode
q = sess\
.query(t)\
.filter(*conditions)\
.order_by(*orderby)\
.distinct()
# For the following countries filter data by features
# for other countries return all data
country_filters = ('BE', )
if self.country_code not in country_filters:
return q
ok_features = set([f.name for f in get_features(self.descriptor)])
out = []
for row in q:
if not self.descriptor.startswith('D1.'):
out.append(row)
continue
feats = set((row.Feature, ))
if feats.intersection(ok_features):
out.append(row)
return out
def descriptor_for_code(self, code):
desc = get_descriptor(code.upper())
return desc
def get_data_old(self):
mapper_class = sql2018.MRUsPublication
mc_mru_descr = sql2018.MarineReportingUnit
res = []
marine_waters_data = self.context._get_marine_waters_data()
marine_waters_total = sum(
[x[2] for x in marine_waters_data if x[0] == self.country_code])
# for better query speed we get only these columns
col_names = ('Country', 'rZoneId', 'thematicId', 'nameTxtInt',
'nameText', 'spZoneType', 'legisSName', 'Area')
columns = [getattr(mapper_class, name) for name in col_names]
count, data = db.get_all_specific_columns(
columns, mapper_class.Country == self.country_code)
mrus_needed = [x.thematicId for x in data]
_, mru_descriptions = db.get_all_specific_columns([
mc_mru_descr.MarineReportingUnitId, mc_mru_descr.Description,
mc_mru_descr.nameTxtInt, mc_mru_descr.nameText
], mc_mru_descr.MarineReportingUnitId.in_(mrus_needed))
_, art8_data = db.get_all_specific_columns([
sql2018.t_V_ART8_GES_2018.c.MarineReportingUnit,
sql2018.t_V_ART8_GES_2018.c.GESComponent
], sql2018.t_V_ART8_GES_2018.c.CountryCode == self.country_code)
_, art9_data = db.get_all_specific_columns([
sql2018.t_V_ART9_GES_2018.c.MarineReportingUnit,
sql2018.t_V_ART9_GES_2018.c.GESComponent
], sql2018.t_V_ART9_GES_2018.c.CountryCode == self.country_code)
art8_art9_data = set(art8_data + art9_data)
for row in data:
mru = row.thematicId
description = [
x.Description
# (x.nameTxtInt is not None and x.nameTxtInt.strip()
# or x.nameText or "")
for x in mru_descriptions if x.MarineReportingUnitId == mru
]
if not description:
description = row.nameTxtInt or row.nameText or ""
else:
description = description[0]
translation = get_translated(description, self.country_code) or ""
area = int(round(row.Area))
if not translation:
retrieve_translation(self.country_code, description)
self._translatable_values.append(description)
prop_water = int(round((area / marine_waters_total) * 100))
descr_list = set(
[x[1] for x in art8_art9_data if x[0] == row.thematicId])
descr_list = sorted(descr_list, key=natural_sort_key)
descr_list_norm = []
for d in descr_list:
try:
desc = get_descriptor(d).template_vars['title']
except:
desc = d
descr_list_norm.append(desc)
descriptors = ', '.join(descr_list_norm)
res.append(
(row.rZoneId, row.spZoneType, mru, description, translation,
'{:,}'.format(area), prop_water, descriptors))
return res
def __call__(self):
alsoProvides(self.request, IDisableCSRFProtection)
if 'assessor' in self.request.form:
assessors = self.request.form['assessor']
if isinstance(assessors, list):
assessors = ', '.join(assessors)
self.context.saved_assessment_data.ass_new = assessors
# BBB:
context = self.context
if not hasattr(context, 'saved_assessment_data') or \
not isinstance(context.saved_assessment_data, PersistentList):
context.saved_assessment_data = AssessmentData()
# Assessment data 2012
descriptor_criterions = get_descriptor(self.descriptor).criterions
country_name = self._country_folder.title
try:
db_data_2012 = get_assessment_data_2012_db(country_name,
self.descriptor,
self.article)
assessments_2012 = filter_assessment_data_2012(
db_data_2012,
self.country_region_code, # TODO: this will need refactor
descriptor_criterions,
)
self.assessment_data_2012 = self.assessment_data_2012_tpl(
data=assessments_2012)
if assessments_2012.get(country_name):
score_2012 = assessments_2012[country_name].score
conclusion_2012 = assessments_2012[country_name].overall_ass
else: # fallback
ctry = assessments_2012.keys()[0]
score_2012 = assessments_2012[ctry].score
conclusion_2012 = assessments_2012[ctry].overall_ass
report_by, assessors, assess_date, source_file = \
get_assessment_head_data_2012(self.article,
self.country_region_code,
self._country_folder.id)
except:
logger.exception("Could not get assessment data for 2012")
self.assessment_data_2012 = ''
score_2012 = 0
conclusion_2012 = 'Not found'
report_by, assessors, assess_date, source_file = [
'Not found'
] * 3 + [('Not found', '')]
# Assessment header 2012
self.assessment_header_2012 = self.assessment_header_template(
report_by=report_by,
assessor_list=[],
assessors=assessors,
assess_date=assess_date,
source_file=source_file,
show_edit_assessors=False,
show_file_version=False,
)
# Assessment data 2018
data = self.context.saved_assessment_data.last()
elements = self.questions[0].get_all_assessed_elements(
self.descriptor_obj, muids=self.muids)
article_weights = ARTICLE_WEIGHTS
assessment = format_assessment_data(self.article, elements,
self.questions, self.muids, data,
self.descriptor_obj,
article_weights, self)
assessment.phase_overall_scores.coherence = self.get_coherence_data(
self.country_region_code, self.descriptor, self.article)
# score_2012 = score_2012
conclusion_2012_color = CONCLUSION_COLOR_TABLE.get(score_2012, 0)
change = (assessment.phase_overall_scores.get_range_index_for_phase(
'adequacy') - score_2012)
# if 2018 adequacy is not relevant, change since 2012 is not relevant
if assessment.phase_overall_scores.adequacy['conclusion'][0] == '-':
change = 'Not relevant (-)'
self.assessment_data_2018_html = self.assessment_data_2018_tpl(
assessment=assessment,
score_2012=score_2012,
conclusion_2012=conclusion_2012,
conclusion_2012_color=conclusion_2012_color,
change_since_2012=change,
can_comment=self.can_comment)
# Assessment header 2018
report_by_2018 = u'Commission'
# assessors_2018 = self.context.saved_assessment_data.assessors
assessors_2018 = getattr(self.context.saved_assessment_data, 'ass_new',
'Not assessed')
assess_date_2018 = data.get('assess_date', u'Not assessed')
source_file_2018 = ('To be addedd...', '.')
can_edit = self.check_permission('wise.msfd: Edit Assessment')
show_edit_assessors = self.assessor_list and can_edit
file_version = self.get_file_version(self.country_date_assessed)
self.assessment_header_2018_html = self.assessment_header_template(
report_by=report_by_2018,
assessor_list=self.assessor_list,
assessors=assessors_2018,
assess_date=assess_date_2018,
source_file=source_file_2018,
show_edit_assessors=show_edit_assessors,
show_file_version=True,
file_version=file_version)
return self.index()
def setup_data(self):
self.article9_cols = self.get_article9_columns()
filename = self.context.get_report_filename()
if not isinstance(filename, tuple):
filename = [filename]
_cols = []
_muids = []
for fname in filename:
text = get_xml_report_data(fname)
if not text:
self.rows = []
return self.template()
root = fromstring(text)
def xp(xpath, node=root):
return node.xpath(xpath, namespaces=NSMAP)
muids = xp('//w:MarineUnitID/text()')
count, res = db.get_marine_unit_id_names(list(set(muids)))
labels = [ItemLabel(m, u'{} ({})'.format(t, m)) for m, t in res]
# special case for PL where marine_unit_ids are not imported into DB
# therefore we cannot get the labels for them
if muids and not labels:
labels = [ItemLabel(m, m) for m in set(muids)]
_muids.extend(labels)
muids = ItemList(labels)
descriptor = get_descriptor(self.descriptor)
self.descriptor_label = descriptor.title
reported = xp("//w:DesriptorCriterionIndicator/text()")
gcs = self.filtered_ges_components(reported)
self.rows = []
# wrap the target per MarineUnitID
all_target_indicators = [
TargetsIndicators(node) for node in xp('w:TargetsIndicators')
]
cols = [
A10Item(self, gc, all_target_indicators, self.country_code,
self.region_code, muids) for gc in gcs
]
_cols.extend(cols)
self.muids_labeled = sorted(_muids,
key=lambda l: natural_sort_key(l.name))
self.cols = _cols
# unwrap the columns into rows
for col in _cols:
for name in col.keys():
values = []
for inner in _cols:
values.append(inner[name])
raw_values = []
vals = []
for v in values:
raw_values.append(v)
vals.append(
self.context.translate_value(name, v,
self.country_code))
row = RawRow(name, vals, raw_values)
self.rows.append(row)
break # only need the "first" row
def get_data(self):
mapper_class = sql2018.MRUsPublication
mc_mru_descr = sql2018.MarineReportingUnit
data = _4GEO_DATA
data_filtered = [
row for row in data if row.MemberState == self.country_code
]
# for better query speed we get only these columns
col_names = ('Country', 'rZoneId', 'thematicId', 'nameTxtInt',
'nameText', 'spZoneType', 'legisSName', 'Area')
columns = [getattr(mapper_class, name) for name in col_names]
_, data_db = db.get_all_specific_columns(
columns, mapper_class.Country == self.country_code)
res = []
_, art8_data = db.get_all_specific_columns([
sql2018.t_V_ART8_GES_2018.c.MarineReportingUnit,
sql2018.t_V_ART8_GES_2018.c.GESComponent
], sql2018.t_V_ART8_GES_2018.c.CountryCode == self.country_code)
_, art9_data = db.get_all_specific_columns([
sql2018.t_V_ART9_GES_2018.c.MarineReportingUnit,
sql2018.t_V_ART9_GES_2018.c.GESComponent
], sql2018.t_V_ART9_GES_2018.c.CountryCode == self.country_code)
art8_art9_data = set(art8_data + art9_data)
for row in data_filtered:
# if row.Status == 'Not used':
# continue
description = row.MRUName
mru_id = self.norm_mru_id(row.MRUID)
translation = get_translated(description, self.country_code) or ''
area = row.MRUArea and int(round(row.MRUArea)) or 0
if not area:
area = [x.Area for x in data_db if x.thematicId == mru_id]
area = area and int(round(area[0])) or 0
if not translation:
retrieve_translation(self.country_code, description)
self._translatable_values.append(description)
# prop_water = int(round((area / marine_waters_total) * 100))
prop_water = (row.MRUCoverage
and "{0:.1f}".format(float(row.MRUCoverage)) or 0)
if not prop_water:
mw_total = [
x.MarineWatersArea for x in data_filtered
if x.MarineWatersArea
]
mw_total = mw_total and mw_total[0] or 0
prop_water = int(round((area / mw_total) * 100))
descr_list = set([x[1] for x in art8_art9_data if x[0] == mru_id])
descr_list = sorted(descr_list, key=natural_sort_key)
descr_list_norm = []
for d in descr_list:
try:
desc = get_descriptor(d).template_vars['title']
except:
desc = d
descr_list_norm.append(desc)
descriptors = ', '.join(descr_list_norm)
res.append(
(row.MarineRegion, row.AssessmentArea, mru_id, description,
translation, '{:,}'.format(area), prop_water, descriptors))
return res
def setup_data(self):
descriptor_class = get_descriptor(self.descriptor)
all_ids = descriptor_class.all_ids()
self.descriptor_label = descriptor_class.title
if self.descriptor.startswith('D1.'):
all_ids.add('D1')
fileurls = self._filename
_mp_nodes = []
_sub_prog_nodes = []
# separate Monitoring Programmes from Sub Programmes
for fileurl in fileurls:
try:
root = self.get_report_file_root(fileurl)
except XMLSyntaxError:
continue
region = xp('//Region', root)
if region:
region = region[0].text
if region not in self.context.regions:
continue
if root.tag == 'MON':
nodes = xp('//MonitoringProgrammes/*', root)
_mp_nodes.extend(nodes)
if root.tag == 'MONSub':
nodes = xp('//SubProgrammes/*', root)
_sub_prog_nodes.extend(nodes)
# filter duplicate MP nodes, only keep the latest
mp_seen = []
mp_nodes = []
for mp_node in _mp_nodes:
if mp_node.tag in mp_seen:
continue
mp_nodes.append(mp_node)
# filter duplicate SubProg nodes, only keep the latest
sub_prog_seen = []
sub_prog_nodes = []
for sub_node in _sub_prog_nodes:
subprog_id = xp('./Q4g_SubProgrammeID/text()', sub_node)
if subprog_id in sub_prog_seen:
continue
sub_prog_nodes.append(sub_node)
items = []
for mp in mp_nodes:
# filter empty nodes
if not mp.getchildren():
continue
# filter mp node by ges criteria
ges_crit = xp('.//Q5a_RelevantGESCriteria', mp)
if ges_crit:
ges_crit_text = ges_crit[0].text
ges_crit = (ges_crit_text and set(ges_crit_text.split(' '))
or set())
if not all_ids.intersection(ges_crit):
continue
subprogrammes = xp('.//ReferenceSubProgramme', mp)
for sub_prog in subprogrammes:
subprog_id = xp('./SubMonitoringProgrammeID/text()', sub_prog)
subprog_id = subprog_id[0].replace('.xml', '').strip()
subp_name = xp('./SubMonitoringProgrammeName/text()', sub_prog)
sub_prog_node = [
x for x in sub_prog_nodes
if xp('./Q4g_SubProgrammeID', x)[0].text == subprog_id
]
sub_prog_node = (len(sub_prog_node) and sub_prog_node[0]
or SUBEMPTY)
item = self._make_item(mp, sub_prog_node, subp_name[0])
items.append(item)
self.rows = []
items = sorted(items,
key=lambda i: [getattr(i, o) for o in self.sort_order])
self.cols = items
self.items_to_rows(items)
def matches_descriptor(self, descriptor):
descriptor = get_descriptor(descriptor)
self_crits = set(self.indicators + self.criterias +
self.criterias_from_topics)
return bool(self_crits.intersection(descriptor.all_ids()))
def matches_descriptor(self, descriptor):
descriptor = get_descriptor(descriptor)
self_crits = set(self.criterias)
return bool(self_crits.intersection(descriptor.all_ids()))
def setup_data(self, filename=None):
if not filename:
filename = self.context.get_report_filename()
if not isinstance(filename, tuple):
filename = [filename]
def filter_descriptors(nodes, descriptor):
res = []
for d in nodes:
rf = x('w:ReportingFeature/text()', d)[0]
rf = criteria_from_gescomponent(rf)
if rf in all_ids:
res.append(d)
return res
descriptor_class = get_descriptor(self.descriptor)
all_ids = descriptor_class.all_ids()
self.descriptor_label = descriptor_class.title
if self.descriptor.startswith('D1.'):
all_ids.add('D1')
_cols = []
_muids = []
for fname in filename:
text = get_xml_report_data(fname)
if not text:
self.rows = []
return self.template()
root = fromstring(text)
def x(xpath, node=root):
return node.xpath(xpath, namespaces=NSMAP)
# these are the records we are interested in
descriptors = filter_descriptors(x('//w:Descriptors'),
self.descriptor)
# the xml file is a collection of records that need to be agregated
# in order to "simplify" their information. For example, the
# ThresholdValues need to be shown for all MarineUnitIds, but the
# grouping criteria is the "GES Component"
cols = []
seen = []
muids = root.xpath('//w:MarineUnitID/text()', namespaces=NSMAP)
count, res = db.get_marine_unit_id_names(list(set(muids)))
muid_ids = [y.id for y in self.muids]
labels = [ItemLabel(m, t or m) for m, t in res if m in muid_ids]
# special case for PL where marine_unit_ids are not imported into DB
# therefore we cannot get the labels for them
if muids and not labels:
labels = [ItemLabel(m, m) for m in set(muids)]
self.muids_labeled = sorted(labels,
key=lambda l: natural_sort_key(l.name))
_muids.extend(labels)
muids = ItemList(labels)
for node in descriptors:
col_id = node.find('w:ReportingFeature', namespaces=NSMAP).text
if col_id in seen:
continue
item = A9Item(self, node, descriptors, muids)
cols.append(item)
seen.append(item.id)
_cols.extend(cols)
_muids = ItemList(_muids)
# insert missing criterions
self.insert_missing_criterions(descriptor_class, _cols, _muids)
self.rows = []
sorted_ges_c = sorted_by_criterion([c.ges_component() for c in _cols])
def sort_func(col):
return sorted_ges_c.index(col.ges_component())
sorted_cols = sorted(_cols, key=sort_func)
self.cols = list(sorted_cols)
for col in sorted_cols:
for name in col.keys():
values = []
for inner in sorted_cols:
values.append(inner[name])
raw_values = []
vals = []
for v in values:
raw_values.append(v)
vals.append(
self.context.translate_value(name, v,
self.country_code))
# values = [self.context.translate_value(name, value=v)
# for v in values]
row = RawRow(name, vals, raw_values)
self.rows.append(row)
break # only need the "first" row
def setup_data(self):
filename = self.context.get_report_filename()
text = get_xml_report_data(filename)
root = fromstring(text)
def xp(xpath, node=root):
return node.xpath(xpath, namespaces=NSMAP)
# TODO: should use declared set of marine unit ids
xml_muids = sorted(set(xp('//w:MarineUnitID/text()')))
self.rows = [
Row('Reporting area(s) [MarineUnitID]',
[', '.join(set(xml_muids))]),
]
report_map = defaultdict(list)
root_tags = get_report_tags(root)
ReportTag = None
# basic algorthim to detect what type of report it is
article = self.article
# override the default translatable
fields = REPORT_DEFS[self.context.year][article]\
.get_translatable_fields()
self.context.TRANSLATABLES.extend(fields)
for name in root_tags:
nodes = xp('//w:' + name)
for node in nodes:
try:
rep = ReportTag(node, NSMAP)
except:
# There are some cases when an empty node is reported
# and the ReportTag class cannot be initialized because
# MarineUnitID element is not present in the node
# see ../fi/bal/d5/art8/@@view-report-data-2012
# search for node MicrobialPathogens
continue
import pdb
pdb.set_trace()
# TODO for D7(maybe for other descriptors too)
# find a way to match the node with the descriptor
# because all reported criterias and indicators are GESOther
if rep.matches_descriptor(self.descriptor):
report_map[rep.marine_unit_id].append(rep)
descriptor = get_descriptor(self.descriptor)
ges_crits = [descriptor] + list(descriptor.criterions)
# a bit confusing code, we have multiple sets of rows, grouped in
# report_data under the marine unit id key.
report_data = {}
# TODO: use reported list of muids per country,from database
for muid in xml_muids:
if muid not in report_map:
logger.warning("MarineUnitID not reported: %s, %s, Article 8",
muid, self.descriptor)
report_data[muid] = []
continue
m_reps = report_map[muid]
if len(m_reps) > 1:
logger.warning(
"Multiple report tags for this "
"marine unit id: %r", m_reps)
rows = []
for i, report in enumerate(m_reps):
# if i > 0: # add a splitter row, to separate reports
# rows.append(Row('', ''))
cols = report.columns(ges_crits)
for col in cols:
for name in col.keys():
values = []
for inner in cols:
values.append(inner[name])
translated_values = [
self.context.translate_value(
name, v, self.country_code) for v in values
]
row = RawRow(name, translated_values, values)
rows.append(row)
break # only need the "first" row, for headers
report_data[muid] = rows
res = {}
muids = {m.id: m for m in self.muids}
for mid, v in report_data.items():
mlabel = muids.get(mid)
if mlabel is None:
logger.warning("Report for non-defined muids: %s", mid)
mid = unicode(mid)
mlabel = MarineReportingUnit(mid, mid)
res[mlabel] = v
# self.muids = sorted(res.keys())
self.rows = res
def descriptor_obj(self):
return get_descriptor(self.descriptor)
def descriptor_obj(self, descriptor):
return get_descriptor(descriptor)
