def group_by_mru(data):
""" Group data by mru
It returns a dict where the keys are ItemLabels of the MRU and values are
list of rows for that mru
"""
if not data:
return {}
res = defaultdict(list)
mrus = {}
# rows with empty MRU field are added to all MRUs
# This case applies for Art9, when justification for delay is reported
rows_extra = []
for row in data:
if not row.MarineReportingUnit: # skip rows without muid
rows_extra.append(row)
continue
mru = row.MarineReportingUnit
if mru not in mrus:
mru_label = GES_LABELS.get('mrus', mru)
if mru_label != mru:
mru_label = u"{} ({})".format(mru_label, unicode(mru))
mru_item = ItemLabel(mru, mru_label)
mrus[mru] = mru_item
else:
mru_item = mrus[mru]
res[mru_item].append(row)
for mru in res.keys():
for row in rows_extra:
res[mru].append(row)
# Special case for Art9, when no real data is reported besides
# justification for delay
if not res and rows_extra:
_mru = ItemLabel('No MRUs', 'No MRUs')
res[_mru].extend(rows_extra)
return res
def get_countries_reports_row(self):
url = self.request['URL0']
reg_main = self._countryregion_folder.id.upper()
subregions = [
r.subregions for r in REGIONAL_DESCRIPTORS_REGIONS
if reg_main in r.code
]
rows = []
country_names = []
template = u'<a style="cursor: pointer;" target="_blank" href="{}">{}</a>'
for country in self.context.available_countries:
value = []
c_code = country[0]
c_name = country[1]
regions = [
r.code for r in REGIONAL_DESCRIPTORS_REGIONS
if len(r.subregions) == 1 and c_code in r.countries
and r.code in subregions[0]
]
for r in regions:
report_url = get_nat_desc_country_url(url, reg_main, c_code, r)
value.append(ItemLabel(r, template.format(report_url, r)))
country_names.append(simple_itemlist(value))
rows.append(('', country_names))
return rows
def row_activity_types(self):
x = 'parent::*/w:Activities/w:Activity/w:ActivityType/text()'
v = set(self.node[x])
labels = [ItemLabel(a, COMMON_LABELS.get(a, a)) for a in v]
return ItemList(labels)
def threshold_value(self):
values = {}
for n in self.siblings:
tv = n['w:ThresholdValue/text()']
if not tv:
continue
muid = n['w:MarineUnitID/text()'][0]
# filter values according to region's marine unit ids
if muid not in self.parent.muids:
continue
values[muid] = tv[0]
rows = []
count, mres = db.get_marine_unit_id_names(values.keys())
muid_labels = dict(mres)
for muid in sorted(values.keys()):
name = u'{} = {}'.format(muid, values[muid])
title = u'{} = {}'.format(muid_labels[muid], values[muid])
label = ItemLabel(name, title)
rows.append(label)
return ItemList(rows=rows)
def setup_data(self):
# setup data to be available at 'features_pressures_data' attr
self.get_features_pressures_data()
data = self.features_pressures_data
out = []
general_pressures = set([
'PresAll',
]) # 'Unknown'
for theme, features_for_theme in self.features_needed:
theme_data = []
theme_general_pressures = set(
[x for x in features_for_theme if x.endswith('All')])
for feature in features_for_theme:
if feature.endswith('All'):
continue
feature_data = []
for country_id, country_name in self.available_countries:
# if pressure is ending with 'All' it applies to all
# features in the current theme
descriptors_rep = []
for x in data:
descr = x[1]
try:
descr_obj = get_descriptor(descr)
descr = descr_obj.template_vars['title']
descr_name = descr_obj.name
except:
descr_name = descr
ccode = x[0]
if ccode != country_id:
continue
feats = set(x[2].split(','))
if (feature in feats
or general_pressures.intersection(feats) or
theme_general_pressures.intersection(feats)):
descriptors_rep.append((descr_name, descr))
descriptors_rep = set(descriptors_rep)
descriptors_rep = [ItemLabel(*x) for x in descriptors_rep]
feature_data.append(ItemListOverview(descriptors_rep))
theme_data.append(
(self.get_feature_short_name(feature), feature_data))
out.append((theme, theme_data))
return out
def csv_ges_labels_inverse_list(field, value, lang):
vals = set(value.split(','))
res = []
for v in vals:
title = GES_LABELS.get(field.label_collection, v)
i = ItemLabel(title, v)
res.append(i)
return ItemList(rows=res)
def features(self):
t = sql.t_MSFD9_Features
count, res = db.get_all_records(
t,
t.c.MSFD9_Descriptor == self.descriptor_item.MSFD9_Descriptor_ID)
vals = list(
set([x.FeatureType
for x in res] + [x.FeaturesPressuresImpacts for x in res]))
# return vals # remove code below when properly integrated
rows = [ItemLabel(v, get_label(v, 'features')) for v in vals]
return ItemList(rows=rows)
def get_indicators(field, value, lang):
value_orig = value
title = GES_LABELS.get('indicators', value)
url = GES_LABELS.get('indicators_url', value)
tr = get_translated(title, lang)
if tr:
value = u"{} ({})".format(value, title)
title = tr
if url != value_orig:
template = u'<a style="cursor: help;" target="_blank" href="{}">{}</a>'
return ItemLabel(value, template.format(url, title))
# if tr:
# value = u"{} ({})".format(value, title)
#
# return ItemLabel(value, tr)
else:
return ItemLabel(value, title)
def data_table(self):
data = self.get_features_pressures_data()
out = []
# general_pressures = set(['PresAll', 'Unknown'])
for activ_theme, activ_features in self.uses_activities_features:
descriptor_type_data = []
for activ_feat in activ_features:
# if activ_feat.endswith('All'):
# continue
features_rep = data.get(activ_feat, set())
descriptor_data = []
# we iterate on all pressures 'Non-indigenous species',
# 'Microbial pathogens' etc. and check if the pressures
# was reported for the current descriptor and feature
for theme, features_for_theme in self.features_needed:
for feature in features_for_theme:
if feature.endswith('All'):
continue
# if pressure is ending with 'All' it applies to all
# features in the current theme
pressures = filter(
lambda i: i.endswith('All') or i == feature,
list(
features_rep.intersection(
set(features_for_theme))))
# These pressures apply to all themes and features
# general_pressures_reported = list(
# general_pressures.intersection(features_rep)
# )
# pressures.extend(general_pressures_reported)
pressures = [
ItemLabel(x, self.get_feature_short_name(x))
for x in pressures
]
descriptor_data.append(ItemListOverview(pressures))
descriptor_type_data.append(
(get_label(activ_feat, 'features'), descriptor_data))
out.append((activ_theme, descriptor_type_data))
return out
def data_table(self):
pressures_data = self.get_features_pressures_data()
env_targets_data = self.get_env_targets_data()
out = []
general_pressures = set(['PresAll', 'Unknown'])
for descr_type, descriptors in DESCRIPTOR_TYPES:
descriptor_type_data = []
for descriptor in descriptors:
features_rep = pressures_data[descriptor]
descriptor_data = []
# we iterate on all pressures 'Non-indigenous species',
# 'Microbial pathogens' etc. and check if the pressures
# was reported for the current descriptor and feature
for theme, features_for_theme in self.features_needed:
for feature in features_for_theme:
if feature.endswith('All'):
continue
# if pressure is ending with 'All' it applies to all
# features in the current theme
pressures = filter(
lambda i: i.endswith('All') or i == feature,
list(
features_rep.intersection(
set(features_for_theme))))
# These pressures apply to all themes and features
general_pressures_reported = list(
general_pressures.intersection(features_rep))
pressures.extend(general_pressures_reported)
targets = set(
sorted([
ItemLabel(get_label(p, 'targets'), p)
for press in pressures
for p in env_targets_data['-'.join((descriptor,
press))]
]))
descriptor_data.append(ItemListOverview(targets))
descriptor_type_data.append(
(self.get_descriptor_title(descriptor), descriptor_data))
out.append((descr_type, descriptor_type_data))
return out
def get_crit_param_row(self):
rows = []
descriptor = self.descriptor_obj
criterions = [descriptor] + descriptor.sorted_criterions()
for crit in criterions:
values = []
for country_code, country_name in self.countries:
data = [
row
for row in self.db_data
if row.CountryCode == country_code
and row.Criteria == crit.id
# or row.GESComponent.split('/')[0] == crit.id)
and row.Parameter
]
value = self.not_rep
parameters = set([
(d.Parameter, d.ParameterOther)
for d in data
])
_vals = []
for param, param_other in parameters:
cnt = len([
r
for r in data
if (r.Parameter == param
and r.ParameterOther == param_other)
])
# filter empty ParametherOther values
_filtered = [
x
for x in (self.get_label_for_value(param), param_other)
if x
]
_vals.append(u"{} ({})".format(" - ".join(_filtered), cnt))
if data:
value = simple_itemlist(_vals)
values.append(value)
rows.append((ItemLabel(crit.name, crit.title), values))
return rows
def feature(self):
# TODO: this needs more work, to aggregate with siblings
res = set()
all_nodes = [s.node for s in self.siblings]
for n in all_nodes:
fpi = n.xpath('w:Features/w:FeaturesPressuresImpacts/text()',
namespaces=NSMAP)
for x in fpi:
res.add(x)
labels = [ItemLabel(k, COMMON_LABELS.get(k, k) or k) for k in res]
return ItemList(labels)
def set_value(self, fieldname, value):
""" Set the field to a specific value, according to set policies
"""
field = None
for f in self.fields:
if f.name == fieldname:
field = f
break
if not field:
# field definition was not found, fallback
setattr(self, fieldname, value)
return
label_collection = field.label_collection
converter_name = field.converter
filter_values = field.filter_values
# assert (label_name or converter), 'Field should be dropped'
if filter_values:
ok_values = getattr(self.report_class, filter_values)
if ok_values:
values = set(value.split(','))
filtered = values.intersection(ok_values)
value = u','.join(filtered)
if converter_name:
assert '.' not in converter_name
converter = getattr(convert, converter_name)
# special convert method, needs the report_class instance to get
# additional info like url, article, descriptor, region etc.
if converter_name.startswith('__'):
value = converter(field, value, self.report_class)
else:
value = converter(field, value, self.report_class.country_code)
elif label_collection:
title = GES_LABELS.get(label_collection, value)
value = ItemLabel(value, title)
setattr(self, fieldname, value)
def features(self):
features = []
for row in self.feat_pres:
v = row.FeatureType
if v:
features.append(v)
v = row.PhysicalChemicalHabitatsFunctionalPressures
features.append(v)
if 'Other' in v and row.Other:
o = row.Other
features.append(o)
res = [ItemLabel(r, get_label(r, 'features')) for r in set(features)]
return ItemList(res)
def get_threshold_source_row(self):
rows = []
threshs = self.get_unique_values('ThresholdValueSource')
descriptor = self.descriptor_obj
criterions = [descriptor] + descriptor.sorted_criterions()
for crit in criterions:
values = []
for country_code, country_name in self.countries:
value = []
data = [
row
for row in self.db_data
if row.CountryCode == country_code
and row.Parameter
]
for threshold_source in threshs:
found = [
x.ThresholdValueSource
for x in data
if x.ThresholdValueSource == threshold_source
and x.Criteria == crit.id
]
if found:
value.append(u"{} ({})".format(
threshold_source, len(found))
)
if not value:
values.append(self.not_rep)
continue
values.append(simple_itemlist(value))
rows.append((ItemLabel(crit.name, crit.title), values))
return rows
def threshold_value(self):
values = {}
for target in self.targets:
muid = target.marine_unit_id
values[muid] = target['w:ThresholdValue/text()'][0]
rows = []
count, mres = db.get_marine_unit_id_names(values.keys())
muid_labels = dict(mres)
for muid in sorted(values.keys()):
name = u'{} = {}'.format(muid, values[muid])
title = u'{} = {}'.format(muid_labels[muid], values[muid])
label = ItemLabel(name, title)
rows.append(label)
return ItemList(rows=rows)
def setup_data(self):
# setup data to be available at 'features_pressures_data' attr
self.get_features_pressures_data()
data = self.features_pressures_data
out = []
for activ_theme, activ_features in self.uses_activities_features:
activ_features = sorted(
activ_features,
key=lambda i: fixedorder_sortkey(i, FEATURES_ORDER))
theme_data = []
for activ_feat in activ_features:
# if activ_feat.endswith('All'):
# continue
activity_data = []
for country_id, country_name in self.available_countries:
pressures = [
x[3] for x in data
if x[0] == country_id and x[2] == activ_feat
]
sorted_press = sorted(
set(pressures),
key=lambda i: fixedorder_sortkey(i, FEATURES_ORDER))
pressures = [
ItemLabel(x, self.get_feature_short_name(x))
for x in sorted_press
]
activity_data.append(
ItemListOverview(pressures, sort=False))
theme_data.append((get_label(activ_feat,
'features'), activity_data))
out.append((activ_theme, theme_data))
return out
def set_value(self, fieldname, value):
""" Set the field to a specific value, according to set policies
"""
field = None
for f in self.fields:
if f.name == fieldname:
field = f
break
if not field:
# field definition was not found, fallback
setattr(self, fieldname, value)
return
label_collection = field.label_collection
converter = field.converter
filter_values = field.filter_values
# assert (label_name or converter), 'Field should be dropped'
if filter_values:
ok_values = getattr(self.report_class, filter_values)
if ok_values:
values = set(value.split(','))
filtered = values.intersection(ok_values)
value = u','.join(filtered)
if converter:
assert '.' not in converter
converter = getattr(convert, converter)
value = converter(field, value, self.report_class.country_code)
elif label_collection:
title = GES_LABELS.get(label_collection, value)
value = ItemLabel(value, title)
setattr(self, fieldname, value)
def get_ges_achiev_row(self):
all_features = self.get_unique_values("Feature")
rows = []
for feature in all_features:
values = []
for country_code, country_name in self.countries:
value = []
data = [
str(row.GESAchieved)
for row in self.db_data
if (row.CountryCode == country_code
and row.GESAchieved
and row.Feature == feature)
]
if not data:
values.append(self.not_rep)
continue
for achiev in set(data):
found = len([x for x in data if x == achiev])
text = u"{0} ({1})".format(
achiev, found
)
value.append(text)
values.append(simple_itemlist(value))
label = self.get_label_for_value(feature)
rows.append((ItemLabel(feature, label), values))
return rows
def __link_to_nat_desc_art11(field, value, self):
url = self.request['URL0']
reg_main = self._countryregion_folder.id.upper()
subregions = [
r.subregions for r in REGIONAL_DESCRIPTORS_REGIONS
if reg_main in r.code
]
res = []
template = u'<a style="cursor: pointer;" target="_blank" href="{}">{}</a>'
c_code = value.split('/')[3].upper()
regions = [
r.code for r in REGIONAL_DESCRIPTORS_REGIONS if len(r.subregions) == 1
and c_code in r.countries and r.code in subregions[0]
]
for r in regions:
report_url = get_nat_desc_country_url(url, reg_main, c_code, r)
res.append(ItemLabel(r, template.format(report_url, r)))
return ItemList(res)
def get_gescomp_row(self):
rows = []
descriptor = self.descriptor_obj
criterions = [descriptor] + descriptor.sorted_criterions()
for crit in criterions:
values = []
for country_code, country_name in self.countries:
orig = []
translated = []
data = set([
row.GESDescription for row in self.db_data
if row.CountryCode == country_code and (
row.GESComponent == crit.id or row.GESComponent.split(
'/')[0] == crit.id) and row.Features
])
if not data:
values.append(self.not_rep)
continue
for ges_descr in data:
transl = get_translated(ges_descr, country_code) or ''
orig.append(ges_descr)
translated.append(transl)
value = (emptyline_separated_itemlist(orig),
emptyline_separated_itemlist(translated))
values.append(value)
rows.append((ItemLabel(crit.id, crit.title), values))
return rows
def inverse_label(field, value, lang):
title = GES_LABELS.get(field.label_collection, value)
item = ItemLabel(title, value)
return item
def row_assessment_topic(self):
if self.assessment:
v = self.assessment.Topic
return ItemLabel(v, COMMON_LABELS.get(v, v))
def setup_data(self, filename=None):
if not filename:
filename = self.context.get_report_filename()
if not isinstance(filename, tuple):
filename = [filename]
def filter_descriptors(nodes, descriptor):
res = []
for d in nodes:
rf = x('w:ReportingFeature/text()', d)[0]
rf = criteria_from_gescomponent(rf)
if rf in all_ids:
res.append(d)
return res
descriptor_class = get_descriptor(self.descriptor)
all_ids = descriptor_class.all_ids()
self.descriptor_label = descriptor_class.title
if self.descriptor.startswith('D1.'):
all_ids.add('D1')
_cols = []
_muids = []
for fname in filename:
text = get_xml_report_data(fname)
if not text:
self.rows = []
return self.template()
root = fromstring(text)
def x(xpath, node=root):
return node.xpath(xpath, namespaces=NSMAP)
# these are the records we are interested in
descriptors = filter_descriptors(x('//w:Descriptors'),
self.descriptor)
# the xml file is a collection of records that need to be agregated
# in order to "simplify" their information. For example, the
# ThresholdValues need to be shown for all MarineUnitIds, but the
# grouping criteria is the "GES Component"
cols = []
seen = []
muids = root.xpath('//w:MarineUnitID/text()', namespaces=NSMAP)
count, res = db.get_marine_unit_id_names(list(set(muids)))
muid_ids = [y.id for y in self.muids]
labels = [ItemLabel(m, t or m) for m, t in res if m in muid_ids]
# special case for PL where marine_unit_ids are not imported into DB
# therefore we cannot get the labels for them
if muids and not labels:
labels = [ItemLabel(m, m) for m in set(muids)]
self.muids_labeled = sorted(labels,
key=lambda l: natural_sort_key(l.name))
_muids.extend(labels)
muids = ItemList(labels)
for node in descriptors:
col_id = node.find('w:ReportingFeature', namespaces=NSMAP).text
if col_id in seen:
continue
item = A9Item(self, node, descriptors, muids)
cols.append(item)
seen.append(item.id)
_cols.extend(cols)
_muids = ItemList(_muids)
# insert missing criterions
self.insert_missing_criterions(descriptor_class, _cols, _muids)
self.rows = []
sorted_ges_c = sorted_by_criterion([c.ges_component() for c in _cols])
def sort_func(col):
return sorted_ges_c.index(col.ges_component())
sorted_cols = sorted(_cols, key=sort_func)
self.cols = list(sorted_cols)
for col in sorted_cols:
for name in col.keys():
values = []
for inner in sorted_cols:
values.append(inner[name])
raw_values = []
vals = []
for v in values:
raw_values.append(v)
vals.append(
self.context.translate_value(name, v,
self.country_code))
# values = [self.context.translate_value(name, value=v)
# for v in values]
row = RawRow(name, vals, raw_values)
self.rows.append(row)
break # only need the "first" row
def consolidate_date_by_mru(data):
""" Takes data (proxies of data) organized by mru and groups them according
to similarity of data (while ignoring the mru of that proxied row)
This is used by the A9 2018 report.
"""
groups = []
# Rows without MRU reported
# This case applies for Art9, when justification for delay is reported
rows_without_mru = []
for obj in chain(*data.values()):
found = False
for group in groups:
# compare only with the first object from a group because
# all objects from a group should contain the same data
first_from_group = group[0]
if proxy_cmp(obj, first_from_group):
group.append(obj)
found = True
if not found:
groups.append([obj])
# regroup the data by mru, now that we found identical rows
regroup = defaultdict(list)
for batch in groups:
# TODO: get a proper MarineUnitID object
mrus = tuple(sorted(set([r.MarineReportingUnit for r in batch])))
if mrus[0] is None:
rows_without_mru.append(batch[0])
continue
regroup[mrus].append(batch[0])
out = {}
# rewrite the result keys to list of MRUs
for mrus, rows in regroup.items():
mrus_labeled = tuple([
ItemLabel(row, u'{} ({})'.format(GES_LABELS.get('mrus', row), row))
for row in mrus
])
label = LabeledItemList(rows=mrus_labeled)
# TODO how to explain better?
# Skip rows from rows_without_mru if the GESComponent exists
# in rows (we do not insert justification delay/non-use
# if the GESComponent has reported data)
# example: ges component D6C3, D6C4
# .../fi/bal/d6/art9/@@view-report-data-2018
ges_comps_with_data = set(x.GESComponent.id for x in rows)
for row_extra in rows_without_mru:
ges_comp = row_extra.GESComponent.id
if ges_comp in ges_comps_with_data:
continue
rows.append(row_extra)
# rows.extend(rows_without_mru)
out[label] = rows
if not regroup and rows_without_mru:
rows = ItemLabel('No Marine unit ID reported',
'No Marine unit ID reported')
label = LabeledItemList(rows=(rows, ))
out[label] = rows_without_mru
return out
def make_item_label(self, value):
return value
return ItemLabel(value, self.get_label_for_value(value))
def row_topic(self):
v = self.assessment_status['parent::*/w:Topic/text()'][0]
return ItemLabel(v, COMMON_LABELS.get(v, v))
def value_to_label(self, value):
if value in COMMON_LABELS:
return ItemLabel(value, COMMON_LABELS[value])
else:
return value
def __init__(self, context, criterion, targets_indicators, country_code,
region_code, muids):
super(A10Item, self).__init__([])
self.context = context
self.criterion = criterion
self.targets_indicators = targets_indicators
self.country_code = country_code
self.region_code = region_code
self.targets = []
# Note: the handling of D1.x indicators is not ideal
if self.is_descriptor:
for ti in targets_indicators:
targets = ti.targets_for_descriptor(self.criterion)
self.targets.extend(targets)
print self.targets
else:
for ti in targets_indicators:
targets = ti.targets_for_criterion(self.criterion)
self.targets.extend(targets)
pick = self.pick
attrs = [
('Feature [Target or Indicator code]', self.criterion),
('GES descriptor, criterion or indicator [GEScomponent]',
self.ges_component()),
('MarineUnitID', muids),
('Method used', self.method_used()),
# ('Feature [Target code]', self.target_code()),
('Description [Targets]', self.description()),
('Threshold value [TargetValue]', self.threshold_value_a9()),
('Reference point type', pick('w:ReferencePointType/text()')),
('Baseline', pick('w:Baseline/text()')),
('Proportion', pick('w:Proportion/text()')),
('Assessment method', pick('w:AssessmentMethod/text()')),
('Development status', pick('w:DevelopmentStatus/text()')),
('Type of target/indicator', pick('w:Type/text()')),
('Timescale', pick('w:TimeScale/text()')),
('Interim or GES target', pick('w:InterimGESTarget/text()')),
('Compatibility with existing targets/indicators',
pick('w:CompatibilityExistingTargets/text()')),
]
feature_pressures = self.get_feature_pressures()
for name in [
'Physical/chemical features', 'Predominant habitats',
'Functional group', 'Pressures'
]:
labels = [
ItemLabel(k, COMMON_LABELS.get(k, k))
for k in feature_pressures[name]
]
attrs.append((name, ItemList(labels)))
for title, value in attrs:
self[title] = value
def setup_data(self):
self.article9_cols = self.get_article9_columns()
filename = self.context.get_report_filename()
if not isinstance(filename, tuple):
filename = [filename]
_cols = []
_muids = []
for fname in filename:
text = get_xml_report_data(fname)
if not text:
self.rows = []
return self.template()
root = fromstring(text)
def xp(xpath, node=root):
return node.xpath(xpath, namespaces=NSMAP)
muids = xp('//w:MarineUnitID/text()')
count, res = db.get_marine_unit_id_names(list(set(muids)))
labels = [ItemLabel(m, u'{} ({})'.format(t, m)) for m, t in res]
# special case for PL where marine_unit_ids are not imported into DB
# therefore we cannot get the labels for them
if muids and not labels:
labels = [ItemLabel(m, m) for m in set(muids)]
_muids.extend(labels)
muids = ItemList(labels)
descriptor = get_descriptor(self.descriptor)
self.descriptor_label = descriptor.title
reported = xp("//w:DesriptorCriterionIndicator/text()")
gcs = self.filtered_ges_components(reported)
self.rows = []
# wrap the target per MarineUnitID
all_target_indicators = [
TargetsIndicators(node) for node in xp('w:TargetsIndicators')
]
cols = [
A10Item(self, gc, all_target_indicators, self.country_code,
self.region_code, muids) for gc in gcs
]
_cols.extend(cols)
self.muids_labeled = sorted(_muids,
key=lambda l: natural_sort_key(l.name))
self.cols = _cols
# unwrap the columns into rows
for col in _cols:
for name in col.keys():
values = []
for inner in _cols:
values.append(inner[name])
raw_values = []
vals = []
for v in values:
raw_values.append(v)
vals.append(
self.context.translate_value(name, v,
self.country_code))
row = RawRow(name, vals, raw_values)
self.rows.append(row)
break # only need the "first" row
