class MockView(BrowserView):
template = Template("templates/example.pt")
interface_override = Template(
"overrides/interface/z3c.jbot.tests.templates.example.pt")
http_override = Template(
"overrides/http/z3c.jbot.tests.templates.example.pt")
https_override = Template(
"overrides/https/z3c.jbot.tests.templates.example.pt")
class Article10Alternate(BaseArticle2012):
template = Template('pt/report-data-a8.pt')
help_text = """ """
@db.use_db_session('2012')
def setup_data(self):
descriptor = get_descriptor(self.descriptor)
ok_ges_ids = set(descriptor.all_ids())
t = sql.MSFD10Target
# muids = [x.id for x in self.muids]
muids = {m.id: m for m in self.muids}
count, res = db.get_all_records(
t,
t.MarineUnitID.in_(muids.keys()),
t.Topic == 'EnvironmentalTarget',
)
by_muid = defaultdict(list)
for target_item in res:
item = A10AlternateItem(target_item, ok_ges_ids)
if item.needed:
by_muid[target_item.MarineUnitID].append(item)
self.rows = {}
for muid, cols in by_muid.items():
rows = []
if not cols:
continue
for name in cols[0].keys():
values = [c[name] for c in cols]
row = Row(name, values)
rows.append(row)
self.rows[muids[muid]] = rows
def __call__(self):
self.setup_data()
return self.template()
class SnapshotSelectForm(Form):
template = Template('../pt/inline-form.pt')
_updated = False
@property
def fields(self):
snaps = getattr(self.context.context, 'snapshots', [])
if snaps:
default = snaps[-1][0]
else:
default = None
dates = [SimpleTerm(x[0], x[0].isoformat(), x[0]) for x in snaps]
field = Choice(title=u'Date of harvest',
__name__='sd',
vocabulary=SimpleVocabulary(dates),
required=False,
default=default)
return Fields(field)
def update(self):
if not self._updated:
Form.update(self)
self._updated = True
@buttonAndHandler(u'View snapshot', name='view')
def apply(self, action):
return
# TODO: make a condition for this button
@buttonAndHandler(u'Harvest new data', name='harvest')
def harvest(self, action):
data = self.context.get_data_from_db()
self.context.context.snapshots.append((datetime.now(), data))
self.request.response.redirect('./@@view-report-data-2018')
class Article10(BaseArticle2012):
""" Article 10 implementation for nation descriptors data
klass(self, self.request, self.country_code, self.descriptor,
self.article, self.muids, self.colspan)
TODO: we should also extract DescriptorCriterionIndicators from the file
How do we show data?
"""
help_text = """
- we identify the filename for the original XML file, by looking at the
MSFD10_Imports table. This is the same file that you can find in the
report table header on this page.
- we download this file from CDR and parse it.
- we lookup all the "<DescriptorCriterionIndicator>" tags in the file and
filter them according to the current descriptor. See
https://raw.githubusercontent.com/eea/wise.msfd/master/src/wise/msfd/data/ges_terms.csv
for the hierarchical definition table of descriptors to indicators and
criterias. To achieve this we do the following:
- we get the assigned country criterion indicators from the MarineDB
2012 database (we use the MSFD_19a_10DescriptiorsCriteriaIndicators
view)
- we use the first part of the criterion indicator to match the
available criterion ids for the current descriptor. This is because
some DescriptorCriterionIndicator are in a format like:
1.2.1-indicator 5.2B
- for each of these DescriptorCriterionIndicator we build a column in the
result table
Notes, problems:
----------------
- the threshold values are not included in the report XML in
the form that they appear in the specification. We take them from the 2012
A9 report, by matching the criterion id.
- the Functional/Pressures/Habitats rows cannot be matched per criterion,
as they are in the spreadsheet, neither by using the database MarineDB or
the XML files. We read the database view
MSFD_19b_10PhysicalChemicalFeatures where we match the target id, country
and region and get a list of ids which we split by type (functional
group, pressure, habitat, etc), but there's no way of matching to the
criterion.
"""
template = Template('pt/report-data-a10.pt')
def get_article9_columns(self):
""" Get the view for Article 9 2012, which contains the
table with the data. This is needed because we show
the Threshold values from article 9
:return: article 9 view
"""
ctx = self.context
art = 'Art9'
filename = ctx.get_report_filename(art)
view = Article9(ctx, ctx.request, ctx.country_code,
ctx.country_region_code, ctx.descriptor, art,
ctx.muids)
view.setup_data(filename)
return view.cols
def filtered_ges_components(self, seed):
""" Returns a list of valid ges criterion indicator targets
Can be something like "1.6.2-indicator 5.2B" or "3.1" or "D1"
"""
descriptor = get_descriptor(self.descriptor)
country_criterions = country_ges_components(self.country_code)
res = set([self.descriptor])
for d_id in descriptor.all_ids():
if d_id in country_criterions:
res.add(d_id)
for crit in set(country_criterions + seed):
crit_id = crit.split('-', 1)[0]
if crit_id in descriptor.all_ids():
res.add(crit)
return sorted_by_criterion(res)
def setup_data(self):
self.article9_cols = self.get_article9_columns()
filename = self.context.get_report_filename()
if not isinstance(filename, tuple):
filename = [filename]
_cols = []
_muids = []
for fname in filename:
text = get_xml_report_data(fname)
if not text:
self.rows = []
return self.template()
root = fromstring(text)
def xp(xpath, node=root):
return node.xpath(xpath, namespaces=NSMAP)
muids = xp('//w:MarineUnitID/text()')
count, res = db.get_marine_unit_id_names(list(set(muids)))
labels = [ItemLabel(m, u'{} ({})'.format(t, m)) for m, t in res]
# special case for PL where marine_unit_ids are not imported into DB
# therefore we cannot get the labels for them
if muids and not labels:
labels = [ItemLabel(m, m) for m in set(muids)]
_muids.extend(labels)
muids = ItemList(labels)
descriptor = get_descriptor(self.descriptor)
self.descriptor_label = descriptor.title
reported = xp("//w:DesriptorCriterionIndicator/text()")
gcs = self.filtered_ges_components(reported)
self.rows = []
# wrap the target per MarineUnitID
all_target_indicators = [
TargetsIndicators(node) for node in xp('w:TargetsIndicators')
]
cols = [
A10Item(self, gc, all_target_indicators, self.country_code,
self.region_code, muids) for gc in gcs
]
_cols.extend(cols)
self.muids_labeled = sorted(_muids,
key=lambda l: natural_sort_key(l.name))
self.cols = _cols
# unwrap the columns into rows
for col in _cols:
for name in col.keys():
values = []
for inner in _cols:
values.append(inner[name])
raw_values = []
vals = []
for v in values:
raw_values.append(v)
vals.append(
self.context.translate_value(name, v,
self.country_code))
row = RawRow(name, vals, raw_values)
self.rows.append(row)
break # only need the "first" row
def __call__(self):
self.setup_data()
return self.template()
def auto_translate(self):
# report_def = REPORT_DEFS[self.year][self.article]
# translatables = report_def.get_translatable_fields()
self.setup_data()
translatables = self.context.TRANSLATABLES
seen = set()
for item in self.cols:
for k in translatables:
value = item[k]
if not isinstance(value, basestring):
continue
if value not in seen:
retrieve_translation(self.country_code, value)
seen.add(value)
return ''
class Article8(BaseArticle2012):
""" Article 8 implementation for nation descriptors data
klass(self, self.request, self.country_code, self.descriptor,
self.article, self.muids, self.colspan)
"""
template = Template('pt/report-data-a8.pt')
help_text = """
- we identify the filename for the original XML file, by looking at the
MSFD10_Imports table. This is the same file that you can find in the
report table header on this page.
- we download this file from CDR and parse it.
- we try to match the type of file to one of two implementation: Article8a
and Article8b.
- We extract all "main" tags from the file and filter those tags that have
matching criterias to the current descriptor, by using theis table:
https://raw.githubusercontent.com/eea/wise.msfd/master/src/wise/msfd/data/ges_terms.csv
We use the <Indicator> tag plus the <CriteriaType> tag to determine which
parent main tag is ok for current descriptor.
For Article8b we lookup the imported assessment in the database MarineDB,
to be able to provide the related indicators. For each such indicator
which we generate a table column.
"""
def setup_data(self):
filename = self.context.get_report_filename()
if not isinstance(filename, tuple):
filename = [filename]
report_map = defaultdict(list)
_xml_muids = []
for fname in filename:
text = get_xml_report_data(fname)
root = fromstring(text)
def xp(xpath, node=root):
return node.xpath(xpath, namespaces=NSMAP)
# TODO: should use declared set of marine unit ids
xml_muids = sorted(set(xp('//w:MarineUnitID/text()')))
_xml_muids.extend(xml_muids)
self.rows = [
Row('Reporting area(s) [MarineUnitID]',
[', '.join(set(xml_muids))]),
]
# each of the following report tags can appear multiple times, once per
# MarineUnitID. Each one can have multiple <AssessmentPI>,
# for specific topics. An AssessmentPI can have multiple indicators
root_tags = get_report_tags(root)
ReportTag = None
# basic algorthim to detect what type of report it is
article = self.article
if 'Nutrients' in root_tags:
ReportTag = ReportTag8b
article = 'Art8b'
elif 'FunctionalGroupFeatures' in root_tags:
ReportTag = ReportTag8a
article = 'Art8a'
if ReportTag is None:
logger.warning("Unhandled report type?")
self.rows = []
return self.template()
# override the default translatable
fields = REPORT_DEFS[self.context.year][article]\
.get_translatable_fields()
self.context.TRANSLATABLES.extend(fields)
for name in root_tags:
nodes = xp('//w:' + name)
for node in nodes:
try:
rep = ReportTag(node, NSMAP)
except:
# There are some cases when an empty node is reported
# and the ReportTag class cannot be initialized because
# MarineUnitID element is not present in the node
# see ../fi/bal/d5/art8/@@view-report-data-2012
# search for node MicrobialPathogens
continue
import pdb
pdb.set_trace()
# TODO for D7(maybe for other descriptors too)
# find a way to match the node with the descriptor
# because all reported criterias and indicators are GESOther
if rep.matches_descriptor(self.descriptor):
report_map[rep.marine_unit_id].append(rep)
descriptor = get_descriptor(self.descriptor)
ges_crits = [descriptor] + list(descriptor.criterions)
# a bit confusing code, we have multiple sets of rows, grouped in
# report_data under the marine unit id key.
report_data = {}
# TODO: use reported list of muids per country,from database
for muid in _xml_muids:
if muid not in report_map:
logger.warning("MarineUnitID not reported: %s, %s, Article 8",
muid, self.descriptor)
report_data[muid] = []
continue
m_reps = report_map[muid]
if len(m_reps) > 1:
logger.warning(
"Multiple report tags for this "
"marine unit id: %r", m_reps)
rows = []
for i, report in enumerate(m_reps):
# if i > 0: # add a splitter row, to separate reports
# rows.append(Row('', ''))
cols = report.columns(ges_crits)
for col in cols:
for name in col.keys():
values = []
for inner in cols:
values.append(inner[name])
translated_values = [
self.context.translate_value(
name, v, self.country_code) for v in values
]
row = RawRow(name, translated_values, values)
rows.append(row)
break # only need the "first" row, for headers
report_data[muid] = rows
res = {}
muids = {m.id: m for m in self.muids}
for mid, v in report_data.items():
mlabel = muids.get(mid)
if mlabel is None:
logger.warning("Report for non-defined muids: %s", mid)
mid = unicode(mid)
mlabel = MarineReportingUnit(mid, mid)
res[mlabel] = v
# self.muids = sorted(res.keys())
self.rows = res
def __call__(self):
self.setup_data()
return self.template()
def auto_translate(self):
self.setup_data()
translatables = self.context.TRANSLATABLES
seen = set()
for table in self.rows.items():
muid, table_data = table
for row in table_data:
if not row:
continue
if row.title not in translatables:
continue
for value in row.raw_values:
if not isinstance(value, basestring):
continue
if value not in seen:
retrieve_translation(self.country_code, value)
seen.add(value)
return ''
class Article34_2018(BaseArticle2012):
""" Implementation for Article 3/4 2018 reported data
"""
year = '2012'
root = None
template = Template('pt/report-data-secondary.pt')
help_text = ""
def __init__(self, context, request, country_code, region_code,
descriptor, article, muids, filename=None,
previous_mrus=None):
# TODO: use previous_mrus to highlight this file MRUs according to edit
# status: added or deleted
super(Article34_2018, self).__init__(context, request, country_code,
region_code, descriptor, article,
muids)
self.filename = filename
self.previous_mrus = previous_mrus
def get_report_file_root(self, filename=None):
if self.root is None:
self.setup_data()
return self.root
def setup_data(self):
filename = self.filename
text = get_xml_report_data(filename)
self.root = fromstring(text)
mru_nodes = xp('//w:GeographicalBoundariesID', self.root)
description = xp('//w:Description', self.root)[0]
# TODO: also send the previous file data
main_node = A34Item_2018_main(
self, self.request, description, mru_nodes, self.previous_mrus
)
self.translatable_extra_data = main_node.get_translatable_extra_data()
self.available_mrus = main_node.available_mrus
self.available_regions = main_node.available_regions
self.rows = []
# TODO: this code needs to be explained. It's hard to understand what
# its purpose is
for name in main_node.keys():
values = []
for inner in [main_node]:
values.append(inner[name])
raw_values = []
vals = []
for v in values:
raw_values.append(v)
vals.append(self.context.translate_value(
name, v, self.country_code))
row = RawRow(name, vals, raw_values)
self.rows.append(row)
self.cols = [main_node]
def __call__(self):
if self.root is None:
self.setup_data()
return self.template()
class A34Item_2018_main(Item):
mrus_template = Template('pt/mrus-table-art34.pt')
TRANSLATABLES_EXTRA = ['MRU Name']
def __init__(self, context, request, description,
mru_nodes, previous_mrus=None):
super(A34Item_2018_main, self).__init__([])
self.description = description
self.context = context
self.request = request
attrs = [
('Member state description', self.member_state_descr),
('Region / subregion description', self.region_subregion),
('Subdivisions', self.subdivisions),
('Marine reporting units description', self.assessment_areas),
]
for title, getter in attrs:
self[title] = getter()
setattr(self, title, getter())
mrus = []
for node in mru_nodes:
item = A34Item_2018_mru(node)
mrus.append(item)
sorted_mrus = sorted(mrus, key=lambda x: x['Marine Reporting Unit'])
self._mrus = sorted_mrus
self.available_mrus = [x['Marine Reporting Unit'] for x in sorted_mrus]
self.available_regions = set(
[x['Region or subregion'] for x in sorted_mrus]
)
self.previous_mrus = previous_mrus or []
item_labels = sorted_mrus and sorted_mrus[0].keys() or ""
sorted_mrus = self.mrus_template(
item_labels=item_labels,
item_values=sorted_mrus,
previous_mrus=self.previous_mrus,
country_code=self.context.country_code,
translate_value=self.translate_value
)
self['MRUs'] = sorted_mrus
setattr(self, 'MRUs', sorted_mrus)
# Region or subregion Member state Area type MRU ID Marine
# reporting unit Marine reporting unit
def get_translatable_extra_data(self):
""" Get the translatable fields from the MRU nodes
:return: a list of values to translate
"""
res = []
for row in self._mrus:
for field in self.TRANSLATABLES_EXTRA:
value = getattr(row, field, None)
if not value:
continue
res.append(value)
return set(res)
def translate_value(self, fieldname, value, source_lang):
is_translatable = fieldname in self.TRANSLATABLES_EXTRA
v = self.context.context.translate_view()
return v.translate(source_lang=source_lang,
value=value,
is_translatable=is_translatable)
def sort_mrus(self, cols):
sorted_cols = sorted(
cols, key=lambda _r: (
_r['Member state'],
_r['Region or subregion'],
_r['Marine Reporting Unit'],
_r['MRU Name']
)
)
return sorted_cols
def member_state_descr(self):
text = xp('w:MemberState/text()', self.description)
return text and text[0] or ''
def region_subregion(self):
text = xp('w:RegionSubregion/text()', self.description)
return text and text[0] or ''
def subdivisions(self):
text = xp('w:Subdivisions/text()', self.description)
return text and text[0] or ''
def assessment_areas(self):
text = xp('w:AssessmentAreas/text()', self.description)
return text and text[0] or ''
class Article34(BaseArticle2012):
""" Article 3 & 4 implementation
klass(self, self.request, self.country_code, self.country_region_code,
self.descriptor, self.article, self.muids)
"""
root = None
year = '2012'
template = Template('pt/report-data-secondary.pt')
help_text = ""
def __init__(self, context, request, country_code, region_code,
descriptor, article, muids, filename):
super(Article34, self).__init__(context, request, country_code,
region_code, descriptor, article,
muids)
self.filename = filename
def sort_cols(self, cols):
sorted_cols = sorted(
cols, key=lambda _r: (
_r['Region / subregion description'],
_r['Area type'],
_r['Marine Reporting Unit']
)
)
return sorted_cols
def setup_data(self):
filename = self.filename
text = get_xml_report_data(filename)
self.root = fromstring(text)
nodes = xp('//w:GeographicalBoundariesID', self.root)
description = xp('//w:Description', self.root)[0]
cols = []
for node in nodes:
item = A34Item(self, node, description)
cols.append(item)
sorted_cols = self.sort_cols(cols)
self.rows = []
for col in sorted_cols:
for name in col.keys():
values = []
for inner in sorted_cols:
values.append(inner[name])
raw_values = []
vals = []
for v in values:
raw_values.append(v)
vals.append(self.context.translate_value(
name, v, self.country_code))
row = RawRow(name, vals, raw_values)
self.rows.append(row)
break # only need the "first" row
self.cols = sorted_cols
def __call__(self):
if self.root is None:
self.setup_data()
return self.template()
class ReportData2018Secondary(ReportData2018):
implements(IReportDataViewSecondary)
descriptor = 'Not linked'
country_region_code = 'No region'
Art3 = Template('pt/report-data-secondary-2018.pt')
Art4 = Template('pt/report-data-secondary-2018.pt')
Art7 = Template('pt/report-data-secondary-2018.pt')
def article_name(self):
get_art_name = super(ReportData2018Secondary, self).article_name
if self.article not in ('Art3', 'Art4'):
return get_art_name()
art_name = '{} & {}'.format(get_art_name('Art3'), get_art_name('Art4'))
return art_name
def get_previus_url(self, grouped_urls, url):
for region, group in grouped_urls.items():
# find the right group for our url
if url not in group:
continue
# if our url is the last from its group, it does not have previous
# file
if group[-1] == url:
return None
url_index = group.index(url)
return group[url_index + 1]
def get_report_metadata_from_view(self, view, filename):
fileurl = get_report_file_url(filename)
root = view.get_report_file_root(filename)
reporters = date = None
try:
reporters = root.attrib['GeneratedBy']
date = root.attrib['CreationDate']
except:
pass
metadata = ReportingInformation2018(fileurl, reporters, date)
return metadata
@property
def report_header_title(self):
article = self.article
if self.article in ('Art3', 'Art4'):
article = 'Art3-4'
title = "Member State report: {} / {}".format(
self.country_name,
article,
)
return title
def get_template(self, article):
article = article.replace('-', '')
template = getattr(self, article, None)
return template
def get_implementation_view(self, filename, prev_filename):
""" In other articles (8, 9, 10) for 2018 year,
we get the data from the DB (MSFD2018_production)
Here instead we will get the data from the report xml from CDR
by initializing and calling the view's class to setup the data
"""
klass = {
'Art7': Article7_2018,
'Art3': Article34_2018,
'Art4': Article34_2018
}.get(self.article)
init_args = [
self, self.request, self.country_code, self.country_region_code,
self.descriptor, self.article, self.muids, filename
]
if self.article in ['Art3', 'Art4'] and prev_filename:
prev_view = klass(self, self.request, self.country_code,
self.country_region_code, self.descriptor,
self.article, self.muids, prev_filename)
prev_view.setup_data()
previous_mrus = prev_view.available_mrus
init_args.append(previous_mrus)
view = klass(*init_args)
view.setup_data()
return view
def auto_translate(self):
self.render_reportdata()
seen = set()
all_translatables = (self.translatable_data +
self.translatable_extra_data)
for value in all_translatables:
if not value:
continue
if not isinstance(value, basestring):
continue
if value not in seen:
retrieve_translation(self.country_code, value)
seen.add(value)
messages = IStatusMessage(self.request)
messages.add(
u"Auto-translation initiated, please refresh "
u"in a couple of minutes",
type=u"info")
url = self.context.absolute_url() + '/@@view-report-data-2018'
return self.request.response.redirect(url)
def get_translatable_data(self, view):
res = []
for row in view.rows:
field_name = row.title
if field_name not in self.TRANSLATABLES:
continue
res.extend(row.raw_values)
return set(res)
def render_reportdata(self):
"""
1. Get all reported files under Article 7 or 3/4
2. Render the data separately for all files
3. Concat the rendered htmls into a single
:return: rendered html
"""
translatable_extra_data = []
translatable_data = []
template = self.get_template(self.article)
urls = get_all_report_filenames(self.country_code, self.article)
rendered_results = []
# identify order of files, grouped by region. If multiple regions are
# reported in a file, then just sort them by envelope release date.
# once sorted, create view for each file. Each view can potentially get
# a reference to the previous file data.
grouped_urls = defaultdict(list)
for url in urls:
view = self.get_implementation_view(url, None)
regions = "-".join(view.available_regions)
grouped_urls[regions].append(url)
for (index, url) in enumerate(urls):
prev_url = self.get_previus_url(grouped_urls, url)
# For article 3/4 2018, the data from previous "version" of the
# file should also be sent. Then it will be possible to identify
# which MRUs have been added/removed
view = self.get_implementation_view(url, prev_url)
translatable_extra_data.extend(view.translatable_extra_data)
translatable_data.extend(self.get_translatable_data(view))
report = self.get_report_metadata_from_view(view, url)
# Report Header
report_by = None
report_date = get_envelope_release_date(url)
if report:
report_by = report.ContactOrganisation
# report_date = report.ReportingDate
res = []
source_file = (url.rsplit('/', 1)[-1], url + '/manage_document')
factsheet = get_factsheet_url(url)
view() # updates the view
data = [Proxy2018(row, self) for row in view.cols]
if self.article == 'Art7':
data_by_mru = group_by_mru(data)
else:
data_by_mru = {'no mru': data}
fields = get_report_definition(self.article).get_fields()
for mru, rows in data_by_mru.items():
_rows = items_to_rows(rows, fields)
res.append((mru, _rows))
report_header = self.report_header_template(
title=(index == 0 and self.report_header_title or ''),
factsheet=factsheet,
# TODO: find out how to get info about who reported
report_by=report_by,
source_file=source_file,
report_due=None,
report_date=report_date.date(),
help_text=self.help_text,
multiple_source_files=False,
show_navigation=index == 0,
)
rendered_results.append(
template(data=res,
report_header=report_header,
show_navigation=False))
self.translatable_extra_data = translatable_extra_data
self.translatable_data = translatable_data
res = "<hr/>".join(rendered_results)
return res or "No data found"
class A34Item_2018_main(Item):
mrus_template = Template('pt/mrus-table-art34.pt')
TRANSLATABLES_EXTRA = ['MRU Name']
def __init__(self,
context,
request,
description,
mru_nodes,
root,
previous_mrus=None,
show_mru_usage=False):
super(A34Item_2018_main, self).__init__([])
self.description = description
self.root = root
self.context = context
self.request = request
attrs = [
('Member state marine waters', self.member_state_descr),
('Region / subregion description', self.region_subregion),
('Subdivisions', self.subdivisions),
('Marine reporting units description', self.assessment_areas),
]
for title, getter in attrs:
self[title] = getter()
setattr(self, title, getter())
mrus = []
for node in mru_nodes:
item = A34Item_2018_mru(node, show_mru_usage)
mrus.append(item)
sorted_mrus = sorted(mrus, key=lambda x: x['Marine Reporting Unit'])
self._mrus = sorted_mrus
self.available_mrus = [x['Marine Reporting Unit'] for x in sorted_mrus]
self.available_regions = set(
[x['Region or subregion'] for x in sorted_mrus])
self.previous_mrus = previous_mrus or []
item_labels = sorted_mrus and sorted_mrus[0].keys() or ""
sorted_mrus = self.mrus_template(
item_labels=item_labels,
item_values=sorted_mrus,
previous_mrus=self.previous_mrus,
country_code=self.context.country_code,
translate_value=self.translate_value)
self['MRUs'] = sorted_mrus
setattr(self, 'MRUs', sorted_mrus)
# Region or subregion Member state Area type MRU ID Marine
# reporting unit Marine reporting unit
cooperation_attrs = [
('Region/ subregion', self.coop_region_subregion()),
('Art. 8 countries involved',
self.coop_by_params('Art8', 'CountriesInvolved')),
('Art. 8 nature of coordination',
self.coop_by_params('Art8', 'NatureCoordination')),
('Art. 8 regional coherence',
self.coop_by_params('Art8', 'RegionalCoherence')),
('Art. 8 regional coherence problems',
self.coop_by_params('Art8', 'RegionalCoordinationProblems')),
('Art. 9 countries involved',
self.coop_by_params('Art9', 'CountriesInvolved')),
('Art. 9 nature of coordination',
self.coop_by_params('Art9', 'NatureCoordination')),
('Art. 9 regional coherence',
self.coop_by_params('Art9', 'RegionalCoherence')),
('Art. 9 regional coherence problems',
self.coop_by_params('Art9', 'RegionalCoordinationProblems')),
('Art. 10 countries involved',
self.coop_by_params('Art10', 'CountriesInvolved')),
('Art. 10 nature of coordination',
self.coop_by_params('Art10', 'NatureCoordination')),
('Art. 10 regional coherence',
self.coop_by_params('Art10', 'RegionalCoherence')),
('Art. 10 regional coherence problems',
self.coop_by_params('Art10', 'RegionalCoordinationProblems')),
]
for title, data in cooperation_attrs:
self[title] = data
setattr(self, title, data)
def get_translatable_extra_data(self):
""" Get the translatable fields from the MRU nodes
:return: a list of values to translate
"""
res = []
for row in self._mrus:
for field in self.TRANSLATABLES_EXTRA:
value = getattr(row, field, None)
if not value:
continue
res.append(value)
return set(res)
def translate_value(self, fieldname, value, source_lang):
is_translatable = fieldname in self.TRANSLATABLES_EXTRA
v = self.context.context.translate_view()
return v.translate(source_lang=source_lang,
value=value,
is_translatable=is_translatable)
def sort_mrus(self, cols):
sorted_cols = sorted(cols,
key=lambda _r:
(_r['Member state'], _r['Region or subregion'],
_r['Marine Reporting Unit'], _r['MRU Name']))
return sorted_cols
def member_state_descr(self):
text = xp('w:MemberState/text()', self.description)
return text and text[0] or ''
def region_subregion(self):
text = xp('w:RegionSubregion/text()', self.description)
return text and text[0] or ''
def subdivisions(self):
text = xp('w:Subdivisions/text()', self.description)
return text and text[0] or ''
def assessment_areas(self):
text = xp('w:AssessmentAreas/text()', self.description)
return text and text[0] or ''
def coop_region_subregion(self):
texts = xp('//w:Cooperation/w:RegionsSubRegions/text()', self.root)
return texts and ' !!! '.join(set(texts)) or ''
def coop_by_params(self, article, node_name):
xpath = '//w:Cooperation[w:Topic = "{}"]' \
'/w:{}/descendant-or-self::*/text()'.format(article, node_name)
texts = xp(xpath, self.root)
return texts and ', '.join(set(sorted(texts))) or ''
class NationalDescriptorSecondaryArticleView(NationalDescriptorArticleView):
""""""
assessment_data_2018_tpl = Template(
'./pt/assessment-data-2018-secondary.pt')
assessment_header_template = Template(
'../pt/assessment-header-secondary.pt')
pdf_assessments = _extract_pdf_assessments()
implements(INationaldescriptorSecondaryArticleView)
_descriptor = 'Not linked'
@property
def country_region_code(self):
return 'No region'
@property
def descriptor_obj(self):
return 'Not linked'
@property
def has_assessment(self):
""" Article 7 will be not assessed, we do not show the 2018 and
2012 assessment tables
"""
if self.article == 'Art7':
return False
return True
def source_pdf_assessment(self):
for row in self.pdf_assessments:
country = row[0]
if country != self.country_code:
continue
article = row[1]
if article != self.article:
continue
url = row[2]
return url
return None
def __call__(self):
alsoProvides(self.request, IDisableCSRFProtection)
if 'assessor' in self.request.form:
assessors = self.request.form['assessor']
if isinstance(assessors, list):
assessors = ', '.join(assessors)
self.context.saved_assessment_data.ass_new = assessors
# BBB:
context = self.context
if not hasattr(context, 'saved_assessment_data') or \
not isinstance(context.saved_assessment_data, PersistentList):
context.saved_assessment_data = AssessmentData()
# Assessment data 2012
# descriptor_criterions = get_descriptor(self.descriptor).criterions
descriptor_criterions = []
country_name = self._country_folder.title
try:
db_data_2012 = get_assessment_data_2012_db(country_name,
self.descriptor,
self.article)
assessments_2012 = filter_assessment_data_2012(
db_data_2012,
self.country_region_code,
descriptor_criterions,
)
self.assessment_data_2012 = self.assessment_data_2012_tpl(
data=assessments_2012)
if assessments_2012.get(country_name):
score_2012 = assessments_2012[country_name].score
conclusion_2012 = assessments_2012[country_name].overall_ass
else: # fallback
ctry = assessments_2012.keys()[0]
score_2012 = assessments_2012[ctry].score
conclusion_2012 = assessments_2012[ctry].overall_ass
report_by, assessors, assess_date, source_file = \
get_assessment_head_data_2012(self.article,
self.country_region_code,
self._country_folder.id)
except:
logger.exception("Could not get assessment data for 2012")
self.assessment_data_2012 = ''
score_2012 = 100
conclusion_2012 = 'Not found'
report_by, assessors, assess_date, source_file = [
'Not found'
] * 3 + [('Not found', '')]
# Assessment header 2012
self.assessment_header_2012 = self.assessment_header_template(
report_by=report_by,
assessor_list=[],
assessors=assessors,
assess_date=assess_date,
source_file=source_file,
show_edit_assessors=False,
show_file_version=False,
)
# Assessment data 2018
data = self.context.saved_assessment_data.last()
elements = self.questions[0].get_all_assessed_elements(
self.descriptor_obj,
country_name=self.country_name,
country_code=self.country_code)
article_weights = ARTICLE_WEIGHTS
assessment = format_assessment_data(self.article, elements,
self.questions, self.muids, data,
self.descriptor_obj,
article_weights, self)
score_2012 = int(round(score_2012))
conclusion_2012_color = CONCLUSION_COLOR_TABLE.get(score_2012, 0)
change = int(
assessment.phase_overall_scores.get_range_index_for_phase(
'adequacy') - score_2012)
self.assessment_data_2018_html = self.assessment_data_2018_tpl(
assessment=assessment,
score_2012=score_2012,
conclusion_2012=conclusion_2012,
conclusion_2012_color=conclusion_2012_color,
change_since_2012=change,
can_comment=self.can_comment)
# Assessment header 2018
report_by_2018 = u'Commission'
# assessors_2018 = self.context.saved_assessment_data.assessors
assessors_2018 = getattr(self.context.saved_assessment_data, 'ass_new',
'Not assessed')
assess_date_2018 = data.get('assess_date', u'Not assessed')
source_file_2018 = ('To be addedd...', '.')
can_edit = self.check_permission('wise.msfd: Edit Assessment')
show_edit_assessors = self.assessor_list and can_edit
self.assessment_header_2018_html = self.assessment_header_template(
report_by=report_by_2018,
assessor_list=self.assessor_list,
assessors=assessors_2018,
assess_date=assess_date_2018,
source_file=source_file_2018,
show_edit_assessors=show_edit_assessors,
show_file_version=False,
)
return self.index()
@property
def title(self):
return u"Commission assessment: {} / {} / 2018".format(
self.country_title,
self.article,
)
class NationalDescriptorArticleView(BaseView, AssessmentDataMixin):
implements(INationaldescriptorArticleView)
section = 'national-descriptors'
assessment_data_2012_tpl = Template('./pt/assessment-data-2012.pt')
assessment_data_2018_tpl = Template('./pt/assessment-data-2018.pt')
year = '2018' # used by self.muids
_questions = NAT_DESC_QUESTIONS
@property
def title(self):
return u"Commission assessment / {} / 2018 / {} / {} / {} ".format(
self.article,
self.descriptor_title,
self.country_title,
self.country_region_name,
)
@property
def criterias(self):
return self.descriptor_obj.sorted_criterions() # criterions
@property
def questions(self):
qs = self._questions.get(self.article, [])
return qs
@db.use_db_session('2018')
def get_file_version(self, date_assessed):
""" Given the assessment date, returns the latest file
"""
edit_url = self._country_folder.absolute_url() + '/edit'
file_name = 'Date assessed not set'
file_url = ''
report_date = 'Not found'
if not date_assessed:
return file_name, edit_url, report_date, edit_url
t = sql2018.ReportedInformation
schemas = {
'Art8': 'ART8_GES',
'Art9': 'ART9_GES',
'Art10': 'ART10_Targets',
}
count, data = db.get_all_records(t,
t.CountryCode == self.country_code,
t.Schema == schemas[self.article],
order_by=t.ReportingDate)
file_name = 'File not found'
for row in data:
if date_assessed >= row.ReportingDate:
file_url = row.ReportedFileLink
report_date = row.ReportingDate
else:
break
if file_url:
file_name = file_url.split('/')[-1]
return file_name, file_url, report_date, edit_url
def __call__(self):
alsoProvides(self.request, IDisableCSRFProtection)
if 'assessor' in self.request.form:
assessors = self.request.form['assessor']
if isinstance(assessors, list):
assessors = ', '.join(assessors)
self.context.saved_assessment_data.ass_new = assessors
# BBB:
context = self.context
if not hasattr(context, 'saved_assessment_data') or \
not isinstance(context.saved_assessment_data, PersistentList):
context.saved_assessment_data = AssessmentData()
# Assessment data 2012
descriptor_criterions = get_descriptor(self.descriptor).criterions
country_name = self._country_folder.title
try:
db_data_2012 = get_assessment_data_2012_db(country_name,
self.descriptor,
self.article)
assessments_2012 = filter_assessment_data_2012(
db_data_2012,
self.country_region_code, # TODO: this will need refactor
descriptor_criterions,
)
self.assessment_data_2012 = self.assessment_data_2012_tpl(
data=assessments_2012)
if assessments_2012.get(country_name):
score_2012 = assessments_2012[country_name].score
conclusion_2012 = assessments_2012[country_name].overall_ass
else: # fallback
ctry = assessments_2012.keys()[0]
score_2012 = assessments_2012[ctry].score
conclusion_2012 = assessments_2012[ctry].overall_ass
report_by, assessors, assess_date, source_file = \
get_assessment_head_data_2012(self.article,
self.country_region_code,
self._country_folder.id)
except:
logger.exception("Could not get assessment data for 2012")
self.assessment_data_2012 = ''
score_2012 = 0
conclusion_2012 = 'Not found'
report_by, assessors, assess_date, source_file = [
'Not found'
] * 3 + [('Not found', '')]
# Assessment header 2012
self.assessment_header_2012 = self.assessment_header_template(
report_by=report_by,
assessor_list=[],
assessors=assessors,
assess_date=assess_date,
source_file=source_file,
show_edit_assessors=False,
show_file_version=False,
)
# Assessment data 2018
data = self.context.saved_assessment_data.last()
elements = self.questions[0].get_all_assessed_elements(
self.descriptor_obj, muids=self.muids)
article_weights = ARTICLE_WEIGHTS
assessment = format_assessment_data(self.article, elements,
self.questions, self.muids, data,
self.descriptor_obj,
article_weights, self)
assessment.phase_overall_scores.coherence = self.get_coherence_data(
self.country_region_code, self.descriptor, self.article)
# score_2012 = score_2012
conclusion_2012_color = CONCLUSION_COLOR_TABLE.get(score_2012, 0)
change = (assessment.phase_overall_scores.get_range_index_for_phase(
'adequacy') - score_2012)
# if 2018 adequacy is not relevant, change since 2012 is not relevant
if assessment.phase_overall_scores.adequacy['conclusion'][0] == '-':
change = 'Not relevant (-)'
self.assessment_data_2018_html = self.assessment_data_2018_tpl(
assessment=assessment,
score_2012=score_2012,
conclusion_2012=conclusion_2012,
conclusion_2012_color=conclusion_2012_color,
change_since_2012=change,
can_comment=self.can_comment)
# Assessment header 2018
report_by_2018 = u'Commission'
# assessors_2018 = self.context.saved_assessment_data.assessors
assessors_2018 = getattr(self.context.saved_assessment_data, 'ass_new',
'Not assessed')
assess_date_2018 = data.get('assess_date', u'Not assessed')
source_file_2018 = ('To be addedd...', '.')
can_edit = self.check_permission('wise.msfd: Edit Assessment')
show_edit_assessors = self.assessor_list and can_edit
file_version = self.get_file_version(self.country_date_assessed)
self.assessment_header_2018_html = self.assessment_header_template(
report_by=report_by_2018,
assessor_list=self.assessor_list,
assessors=assessors_2018,
assess_date=assess_date_2018,
source_file=source_file_2018,
show_edit_assessors=show_edit_assessors,
show_file_version=True,
file_version=file_version)
return self.index()
class Article7(BaseArticle2012):
""" Article 7 implementation for 2012 year
klass(self, self.request, country_code, region_code,
descriptor, article, muids)
"""
template = Template('pt/report-data-secondary.pt')
help_text = ""
def get_report_file_root(self):
filename = self.context.get_report_filename()
text = get_xml_report_data(filename)
root = fromstring(text)
return root
def _make_item(self, node):
item = A7Item(self, node)
return item
def setup_data(self):
root = self.get_report_file_root()
# basic algorthim to detect what type of report it is
article = self.article
# override the default translatable
fields = REPORT_DEFS[self.context.year][article]\
.get_translatable_fields()
self.context.TRANSLATABLES.extend(fields)
cols = []
nodes = xp('//w:CompetentAuthority', root)
for node in nodes:
# filter empty nodes
if not node.getchildren():
continue
item = self._make_item(node)
cols.append(item)
self.rows = []
for col in cols:
for name in col.keys():
values = []
for inner in cols:
values.append(inner[name])
raw_values = []
vals = []
for v in values:
raw_values.append(v)
vals.append(
self.context.translate_value(name, v,
self.country_code))
# values = [self.context.translate_value(name, value=v)
# for v in values]
row = RawRow(name, vals, raw_values)
self.rows.append(row)
break # only need the "first" row
self.cols = cols
def __call__(self):
self.setup_data()
return self.template()
def auto_translate(self):
try:
self.setup_data()
except AssertionError:
return
translatables = self.context.TRANSLATABLES
seen = set()
for row in self.rows:
if not row:
continue
if row.title not in translatables:
continue
for value in row.raw_values:
if not isinstance(value, basestring):
continue
if value not in seen:
retrieve_translation(self.country_code, value)
seen.add(value)
return ''
class ReportData2018(BaseView):
implements(IReportDataView)
report_year = '2018' # used by cache key
year = '2018' # used in report definition and translation
section = 'national-descriptors'
help_texts = {
'Art8': """
The data is retrieved from the MSFD2018_production.V_ART8_GES_2018 database
view, filtered by country code and ges component ids. If the current Descriptor
starts with 'D1.', we also append the 'D1' descriptor to the GES Component ids.
We use this table for the list of GES Components and the descriptor that they
belong to:
https://raw.githubusercontent.com/eea/wise.msfd/master/src/wise/msfd/data/ges_terms.csv
""",
'Art9': """
The data is retrieved from the MSFD2018_production.V_ART9_GES_2018 database
view, filtered by country code and ges component ids. If the current Descriptor
starts with 'D1.', we also append the 'D1' descriptor to the GES Component ids.
We use this table for the list of GES Components and the descriptor that they
belong to:
https://raw.githubusercontent.com/eea/wise.msfd/master/src/wise/msfd/data/ges_terms.csv
""",
'Art10': """
The data is retrieved from the MSFD2018_production.V_ART10_Targets_2018
database view. Because the GESComponent column is not reliable (the Netherlands
reported using the 1.1.3 GESComponent for all their records), we filter the
data using the Parameters and Features available for the current descriptor.
We use this file for the Descriptor to Parameters and Features association
table:
https://svn.eionet.europa.eu/repositories/Reportnet/Dataflows/MarineDirective/MSFD2018/Webforms/msfd2018-codelists.json
""",
'Art3': "To be completed...",
'Art4': "To be completed...",
'Art7': "To be completed..."
}
@property
def help_text(self):
return self.help_texts[self.article]
Art8 = Template('pt/report-data-multiple-muid.pt')
Art9 = Template('pt/report-data-multiple-muid.pt')
Art10 = Template('pt/report-data-multiple-muid.pt')
# Art9 = Template('pt/report-data-single-muid.pt')
subform = None # used for the snapshot selection form
@property
def all_descriptor_ids(self):
descr_class = get_descriptor(self.descriptor)
all_ids = list(descr_class.all_ids())
if self.descriptor.startswith('D1.'):
all_ids.append('D1')
all_ids = set(all_ids)
return all_ids
def _get_order_cols_Art8(self, descr):
descr = descr.split('.')[0]
criteria_priority = ('MarineReportingUnit', 'GESComponent', 'Criteria',
'Feature', 'Element', 'Element2', 'Element2Code',
'IntegrationRuleTypeParameter')
default = (
'MarineReportingUnit',
'GESComponent',
'Feature',
'Element',
'Element2',
'Element2Code',
'Criteria',
'IntegrationRuleTypeParameter',
)
order_by = {
'D2':
criteria_priority,
'D4':
criteria_priority,
'D5': (
'MarineReportingUnit',
'GESComponent',
'Feature',
'Criteria',
'Element',
'Element2',
'Element2Code',
'IntegrationRuleTypeParameter',
),
'D6':
default,
'D7':
criteria_priority,
'D8':
criteria_priority,
'D11':
criteria_priority,
'default':
default
}
return order_by.get(descr, order_by['default'])
def _get_order_cols_Art10(self):
order = ('TargetCode', 'Features', 'Element')
return order
def get_data_from_view_Art8(self):
sess = db.session()
t = sql2018.t_V_ART8_GES_2018
descr_class = get_descriptor(self.descriptor)
all_ids = list(descr_class.all_ids())
if self.descriptor.startswith('D1.'):
all_ids.append('D1')
# muids = [x.id for x in self.muids]
conditions = [
t.c.CountryCode == self.country_code,
# t.c.Region == self.country_region_code,
# t.c.MarineReportingUnit.in_(muids), #
t.c.GESComponent.in_(all_ids)
]
# Handle the case of Romania that submitted duplicate data,
# where Element is empty, but Criteria has data
if self.country_code != 'RO':
conditions.append(
or_(t.c.Element.isnot(None), t.c.Criteria.isnot(None)))
else:
conditions.append(t.c.Element.isnot(None))
if self.country_code != 'DK':
conditions.insert(1, t.c.Region == self.country_region_code)
else:
# Handle the case of Denmark that have submitted a lot of
# information under the DK-TOTAL MRU, which doesn't have a region
# attached.
conditions.insert(
1,
or_(t.c.Region == 'NotReported',
t.c.Region == self.country_region_code))
orderby = [
getattr(t.c, x) for x in self._get_order_cols_Art8(self.descriptor)
]
# groupby IndicatorCode
q = sess\
.query(t)\
.filter(*conditions)\
.order_by(*orderby)\
.distinct()
# For the following countries filter data by features
# for other countries return all data
country_filters = ('BE', )
if self.country_code not in country_filters:
return q
ok_features = set([f.name for f in get_features(self.descriptor)])
out = []
for row in q:
if not self.descriptor.startswith('D1.'):
out.append(row)
continue
feats = set((row.Feature, ))
if feats.intersection(ok_features):
out.append(row)
return out
def get_data_from_view_Art10(self):
t = sql2018.t_V_ART10_Targets_2018
conditions = [t.c.CountryCode == self.country_code]
if self.country_code != 'DK':
conditions.insert(1, t.c.Region == self.country_region_code)
else:
# Handle the case of Denmark that have submitted a lot of
# information under the DK-TOTAL MRU, which doesn't have a region
# attached.
conditions.insert(
1,
or_(t.c.Region == 'NotReported',
t.c.Region == self.country_region_code))
count, res = db.get_all_records_ordered(t,
self._get_order_cols_Art10(),
*conditions)
out = []
# GESComponents contains multiple values separated by comma
# filter rows by splitting GESComponents
for row in res:
ges_comps = getattr(row, 'GESComponents', ())
ges_comps = set([g.strip() for g in ges_comps.split(',')])
if ges_comps.intersection(self.all_descriptor_ids):
out.append(row)
if not self.descriptor.startswith('D1.'):
return out
# conditions = []
# params = get_parameters(self.descriptor)
# p_codes = [p.name for p in params]
# conditions.append(t.c.Parameter.in_(p_codes))
# Filtering results based on FeaturesSmart and other conditions
# I don't think this code should be kept. Probably the edge case should
# be documented. It makes it fragile and dependent on correct
# definitions in FeaturesSmart. I think it's trying to avoid showing
# too many results when the GESComponent has been incorectly reported
# on the <Target> records.
ok_features = set([f.name for f in get_features(self.descriptor)])
out_filtered = []
for row in out:
# Because some Features are missing from FeaturesSmart
# we consider 'D1' descriptor valid for all 'D1.x'
# and we keep the data if 'D1' is present in the GESComponents
# countries_filter = for these countries DO NOT filter by features
ges_comps = getattr(row, 'GESComponents', ())
countries_filter = ('RO', 'DK', 'CY', 'MT')
if 'D1' in ges_comps and self.country_code not in countries_filter:
out_filtered.append(row)
continue
feats = set(row.Features.split(','))
if feats.intersection(ok_features):
out_filtered.append(row)
return out_filtered
def get_data_from_view_Art9(self):
t = sql2018.t_V_ART9_GES_2018
descriptor = get_descriptor(self.descriptor)
all_ids = list(descriptor.all_ids())
if self.descriptor.startswith('D1.'):
all_ids.append('D1')
conditions = [
t.c.CountryCode == self.country_code,
t.c.GESComponent.in_(all_ids)
]
if self.country_code != 'DK':
conditions.insert(
1,
or_(t.c.Region == self.country_region_code,
t.c.Region.is_(None)))
else:
# Handle the case of Denmark that have submitted a lot of
# information under the DK-TOTAL MRU, which doesn't have a region
# attached.
conditions.insert(
1,
or_(t.c.Region == 'NotReported',
t.c.Region == self.country_region_code,
t.c.Region.is_(None)))
count, q = db.get_all_records_ordered(t, ('GESComponent', ),
*conditions)
ok_features = set([f.name for f in get_features(self.descriptor)])
out = []
# There are cases when justification for delay is reported
# for a ges component. In these cases region, mru, features and
# other fields are empty. Justification for delay should be showed
# for all regions, mrus
for row in q:
if not row.Features:
out.append(row)
continue
if not self.descriptor.startswith('D1.'):
out.append(row)
continue
feats = set(row.Features.split(','))
if feats.intersection(ok_features):
out.append(row)
return out
def get_data_from_view(self, article):
data = getattr(self, 'get_data_from_view_' + article)()
return data
@db.use_db_session('2018')
@timeit
def get_data_from_db(self):
data = self.get_data_from_view(self.article)
data = [Proxy2018(row, self) for row in data]
if self.request.form.get('split-mru') and (len(data) > 2000):
if self.muids:
if getattr(self, 'focus_muid', None) is None:
self.focus_muid = self.muids[0].name
self.focus_muids = self._get_muids_from_data(data)
if self.article == 'Art8':
order = self._get_order_cols_Art8(self.descriptor)
data = consolidate_singlevalue_to_list(
data,
'IndicatorCode',
order,
)
data_by_mru = group_by_mru(data)
if self.article == 'Art10':
# data_by_mru = group_by_mru(data)
order = self._get_order_cols_Art10()
data_by_mru = consolidate_singlevalue_to_list(
data, 'MarineReportingUnit', order)
if data_by_mru:
data_by_mru = {"": data_by_mru}
else:
data_by_mru = {}
if self.article == 'Art9':
# data_by_mru = consolidate_date_by_mru(data_by_mru)
data_by_mru = consolidate_singlevalue_to_list(
data, 'MarineReportingUnit')
if data_by_mru:
data_by_mru = {"": data_by_mru}
else:
data_by_mru = {}
insert_missing_criterions(data_by_mru, self.descriptor_obj)
res = []
fields = get_report_definition(self.article).get_fields()
for mru, rows in data_by_mru.items():
_rows = items_to_rows(rows, fields)
res.append((mru, _rows))
# resort the results by marine reporting unit
res_sorted = sorted(res,
key=lambda r: natural_sort_key(r[0].__repr__()))
return res_sorted
def get_snapshots(self):
""" Returns all snapshots, in the chronological order they were created
"""
# TODO: fix this. I'm hardcoding it now to always use generated data
db_data = self.get_data_from_db()
snapshot = (datetime.now(), db_data)
return [snapshot]
# snapshots = getattr(self.context, 'snapshots', None)
#
# if snapshots is None:
# self.context.snapshots = PersistentList()
#
# db_data = self.get_data_from_db()
# snapshot = (datetime.now(), db_data)
#
# self.context.snapshots.append(snapshot)
# self.context.snapshots._p_changed = True
#
# self.context._p_changed = True
#
# return self.context.snapshots
#
# return snapshots
def get_report_data(self):
""" Returns the data to display in the template
Returns a list of "rows (tuples of label: data)"
"""
snapshots = self.get_snapshots()
self.subform.update()
fd, errors = self.subform.extractData()
date_selected = fd['sd']
data = snapshots[-1][1]
if hasattr(self, 'focus_muid'):
# filter the data based on selected muid
# this is used to optmize display of really long data
data = [t for t in data if t[0].name == self.focus_muid]
if date_selected:
filtered = [x for x in snapshots if x[0] == date_selected]
if filtered:
date, data = filtered[0]
else:
raise ValueError("Snapshot doesn't exist at this date")
return data
def _get_muids_from_data(self, data):
muids = set()
for row in data:
o = getattr(row, '__o')
muid = o.MarineReportingUnit
muids.add(muid)
return list(sorted(muids))
# def get_muids_from_data(self, data):
# # TODO: this shouldn't exist anymore
# if isinstance(data[0][0], (unicode, str)):
# all_muids = sorted(set([x[0] for x in data]))
#
# return ', '.join(all_muids)
#
# all_muids = [x[0] for x in data]
# seen = []
# muids = []
#
# for muid in all_muids:
# name = muid.name
#
# if name in seen:
# continue
#
# seen.append(name)
# muids.append(muid)
#
# return ItemList(rows=muids)
@db.use_db_session('2018')
@timeit
def get_report_metadata(self):
""" Returns metadata about the reported information
"""
t = sql2018.ReportedInformation
schemas = {
'Art8': 'ART8_GES',
'Art9': 'ART9_GES',
'Art10': 'ART10_Targets',
}
count, item = db.get_item_by_conditions(
t,
'ReportingDate',
t.CountryCode == self.country_code,
t.Schema == schemas[self.article],
reverse=True,
)
return item
@property
def report_header_title(self):
title = "Member State report / {} / 2018 / {} / {} / {}".format(
self.article,
self.descriptor_title,
self.country_name,
self.country_region_name,
)
return title
def get_report_header(self):
report = self.get_report_metadata()
link = report_by = report_date = None
if report:
link = report.ReportedFileLink
link = (link.rsplit('/', 1)[1], link)
report_by = report.ContactOrganisation
report_date = report.ReportingDate
report_header = self.report_header_template(
title=self.report_header_title,
factsheet=None,
# TODO: find out how to get info about who reported
report_by=report_by,
source_file=link,
report_due='2018-10-15',
report_date=report_date,
help_text=self.help_text,
multiple_source_files=False)
return report_header
@cache(get_reportdata_key, dependencies=['translation'])
def render_reportdata(self):
logger.info("Quering database for 2018 report data: %s %s %s %s",
self.country_code, self.country_region_code, self.article,
self.descriptor)
data = self.get_report_data()
report_header = self.get_report_header()
template = self.get_template(self.article)
return template(data=data, report_header=report_header)
def data_to_xls(self, data):
# Create a workbook and add a worksheet.
out = BytesIO()
workbook = xlsxwriter.Workbook(out, {'in_memory': True})
for index, (wtitle, wdata) in enumerate(data):
_wtitle = '{}_{}'.format(index + 1, unicode(wtitle)[:28])
worksheet = workbook.add_worksheet(_wtitle)
for i, (row_label, row_values) in enumerate(wdata):
worksheet.write(i, 0, row_label.title)
for j, v in enumerate(row_values):
v = unicode(v) or ''
transl = get_translated(v, self.country_code) or v
worksheet.write(i, j + 1, transl)
workbook.close()
out.seek(0)
return out
def download(self):
xlsdata = self.get_report_data()
xlsio = self.data_to_xls(xlsdata)
sh = self.request.response.setHeader
sh(
'Content-Type', 'application/vnd.openxmlformats-officedocument.'
'spreadsheetml.sheet')
fname = "-".join([
self.country_code, self.country_region_code, self.article,
self.descriptor
])
sh('Content-Disposition', 'attachment; filename=%s.xlsx' % fname)
return xlsio.read()
def auto_translate(self, data=None):
if not data:
data = self.get_report_data()
# report_def = REPORT_DEFS[self.year][self.article]
# translatables = report_def.get_translatable_fields()
translatables = self.TRANSLATABLES
seen = set()
for table in data:
muid, table_data = table
for row in table_data:
field, cells = row
if field.name in translatables:
for value in cells:
if value not in seen:
retrieve_translation(self.country_code, value)
seen.add(value)
messages = IStatusMessage(self.request)
messages.add(
u"Auto-translation initiated, please refresh "
u"in a couple of minutes",
type=u"info")
url = self.context.absolute_url() + '/@@view-report-data-2018'
return self.request.response.redirect(url)
def get_template(self, article):
template = getattr(self, article, None)
return template
@timeit
def __call__(self):
# allow focusing on a single muid if the data is too big
if 'focus_muid' in self.request.form:
self.focus_muid = self.request.form['focus_muid'].strip()
# self.focus_muid = 'BAL-AS-EE-ICES_SD_29'
self.content = ''
template = self.get_template(self.article)
if not template:
return self.index()
self.subform = self.get_form()
if ('download' in self.request.form): # and report_data
return self.download()
if 'translate' in self.request.form and self.can_view_assessment_data:
return self.auto_translate()
trans_edit_html = self.translate_view()()
print "will render report"
report_html = self.render_reportdata()
self.report_html = report_html + trans_edit_html
@timeit
def render_html():
return self.index()
return render_html()
def get_form(self):
if not self.subform:
form = SnapshotSelectForm(self, self.request)
self.subform = form
return self.subform
def __call__(self):
""" Article 3 & 4 reports are separated per regions
This means we can have more than one report xml for a country
one for each region
Merge the data from each region, and display it in one table
"""
# we treat Art 3 & 4 different because of multiple report files
if self.article not in ('Art3', 'Art4'):
return super(ReportData2012Secondary, self).__call__()
template = Template('pt/report-data-view-art34.pt')
report_header_template = Template('pt/report-data-header-art34.pt')
regions = get_regions_for_country(self.country_code)
filenames = [(r[0], r[1],
get_report_filename('2012', self.country_code, r[0],
self.article, self.descriptor))
for r in regions]
filenames = sorted(filenames,
key=lambda i: ordered_regions_sortkey(i[0]))
trans_edit_html = self.translate_view()()
reports = []
report_data = []
for region, region_name, filename in filenames:
if not filename:
continue
url = get_report_file_url(filename)
source_file = (filename, url + '/manage_document')
factsheet = get_factsheet_url(url)
view = Article34(self, self.request, self.country_code, region,
self.descriptor, self.article, self.muids,
filename)
rendered_view = view()
rep_info = self.get_reporting_information(filename=filename)
report_header_data = self.get_report_header_data(
rep_info.reporters, source_file, factsheet,
rep_info.report_date)
report_header = report_header_template(self,
self.request,
region=region_name,
**report_header_data)
reports.append(report_header + rendered_view + trans_edit_html)
report_data.append(
(region, serialize_rows(view.rows), report_header_data))
self.reports = reports
if 'download' in self.request.form:
return self.download_art7(report_data)
return template(self, self.request)
class Article11(BaseArticle2012):
""" Article 11 implementation for 2014 year
klass(self, self.request, country_code, region_code,
descriptor, article, muids)
1. Get the report filename with a sparql query
2. With the filename get the report url from CDR
3. Get the data from the xml file
"""
# template = Template('pt/report-data-secondary.pt')
template = Template('pt/report-data-art11.pt')
help_text = ""
available_regions = []
translatable_extra_data = []
is_regional = False
def __init__(self,
context,
request,
country_code,
region_code,
descriptor,
article,
muids,
filenames=None):
super(Article11,
self).__init__(context, request, country_code, region_code,
descriptor, article, muids)
self._filename = filenames
@property
def sort_order(self):
order = ('MonitoringProgrammeName', 'Q4g_SubProgrammeID')
return order
def get_report_filename(self):
if self._filename:
return self._filename
filename = get_report_filename(
'2014',
self.country_code,
self.region_code,
self.article,
self.descriptor,
)
return filename
def get_report_file_root(self, filename=None):
if not filename:
filename = self.get_report_filename()
text = get_xml_report_data(filename)
root = fromstring(text)
return root
def _make_item(self, mp_node, subprog_node, subprog_name):
item = A11Item(self, mp_node, subprog_node, subprog_name)
return item
def auto_translate(self):
try:
self.setup_data()
except AssertionError:
return
translatables = self.context.TRANSLATABLES
seen = set()
for row in self.rows:
if not row:
continue
if row.field.name not in translatables:
continue
for value in row.raw_values:
if not isinstance(value, basestring):
continue
if value not in seen:
retrieve_translation(self.country_code, value)
seen.add(value)
return ''
def items_to_rows(self, items):
rep_fields = self.context.get_report_definition()
for field in rep_fields:
field_name = field.name
values = []
for inner in items:
values.append(inner[field_name])
raw_values = []
vals = []
for v in values:
raw_values.append(v)
vals.append(
self.context.translate_value(field_name, v,
self.country_code))
row = national_compoundrow(self.context, field, vals, raw_values)
self.rows.append(row)
def setup_data(self):
descriptor_class = get_descriptor(self.descriptor)
all_ids = descriptor_class.all_ids()
self.descriptor_label = descriptor_class.title
if self.descriptor.startswith('D1.'):
all_ids.add('D1')
fileurls = self._filename
_mp_nodes = []
_sub_prog_nodes = []
# separate Monitoring Programmes from Sub Programmes
for fileurl in fileurls:
try:
root = self.get_report_file_root(fileurl)
except XMLSyntaxError:
continue
region = xp('//Region', root)
if region:
region = region[0].text
if region not in self.context.regions:
continue
if root.tag == 'MON':
nodes = xp('//MonitoringProgrammes/*', root)
_mp_nodes.extend(nodes)
if root.tag == 'MONSub':
nodes = xp('//SubProgrammes/*', root)
_sub_prog_nodes.extend(nodes)
# filter duplicate MP nodes, only keep the latest
mp_seen = []
mp_nodes = []
for mp_node in _mp_nodes:
if mp_node.tag in mp_seen:
continue
mp_nodes.append(mp_node)
# filter duplicate SubProg nodes, only keep the latest
sub_prog_seen = []
sub_prog_nodes = []
for sub_node in _sub_prog_nodes:
subprog_id = xp('./Q4g_SubProgrammeID/text()', sub_node)
if subprog_id in sub_prog_seen:
continue
sub_prog_nodes.append(sub_node)
items = []
for mp in mp_nodes:
# filter empty nodes
if not mp.getchildren():
continue
# filter mp node by ges criteria
ges_crit = xp('.//Q5a_RelevantGESCriteria', mp)
if ges_crit:
ges_crit_text = ges_crit[0].text
ges_crit = (ges_crit_text and set(ges_crit_text.split(' '))
or set())
if not all_ids.intersection(ges_crit):
continue
subprogrammes = xp('.//ReferenceSubProgramme', mp)
for sub_prog in subprogrammes:
subprog_id = xp('./SubMonitoringProgrammeID/text()', sub_prog)
subprog_id = subprog_id[0].replace('.xml', '').strip()
subp_name = xp('./SubMonitoringProgrammeName/text()', sub_prog)
sub_prog_node = [
x for x in sub_prog_nodes
if xp('./Q4g_SubProgrammeID', x)[0].text == subprog_id
]
sub_prog_node = (len(sub_prog_node) and sub_prog_node[0]
or SUBEMPTY)
item = self._make_item(mp, sub_prog_node, subp_name[0])
items.append(item)
self.rows = []
items = sorted(items,
key=lambda i: [getattr(i, o) for o in self.sort_order])
self.cols = items
self.items_to_rows(items)
def __call__(self):
self.setup_data()
return self.template(data=self.rows)
class Article8Alternate(BaseArticle2012):
template = Template('pt/report-data-a8.pt')
help_text = """ """
implementations = {
'D1': [
A8aSpecies,
A8aFunctional,
],
'D1/D6': [
A8aHabitat
],
'D2': [
A8bNIS,
],
'D3': [
A8bExtractionFishShellfish,
A8bExtractionSeaweedMaerlOther,
],
'D4': [
A8aEcosystem,
A8aPhysical,
A8bAcidification,
],
'D5': [
A8bNutrient,
],
'D6': [
A8bPhysicalDamage
],
'D7': [
A8bHydrologicalProcessess
],
'D8': [
A8bPollutantEvents,
A8bHazardousSubstancesD8
],
'D9': [
A8bHazardousSubstancesD9,
A8bMicrobialPathogens
],
'D10': [
A8bLitter
],
'D11': [
A8bNoise
],
}
def setup_data(self):
# descriptor = get_descriptor(self.descriptor)
# ok_ges_ids = descriptor.all_ids()
descriptor = self.descriptor
if descriptor.startswith('D1.'):
descriptor = 'D1' # TODO: handle other cases
self.rows = defaultdict(list)
# {muid: {field_name: [values, ...], ...}
res = defaultdict(lambda: OrderedDefaultDict())
muids = {m.id: m for m in self.muids}
for Klass in self.implementations[descriptor]:
print 'Started Klass: %s' % (Klass.__name__)
# Klass = self.implementations[descriptor][0]
# count, res = db.get_all_records(
# Impl.mapper,
# Impl.mapper.MarineUnitID.in_(self.muids)
# )
by_muid = defaultdict(list)
for item in Klass.items(self.descriptor, muids.keys()):
by_muid[item.MarineUnitID].append(item)
for muid, cols in by_muid.items():
rows = []
if not cols:
continue
for name in cols[0].keys():
values = [c[name] for c in cols]
res[muid][name].extend(values)
for muid, rows in res.items():
for name, values in rows.items():
row = Row(name, values)
self.rows[muids[muid]].append(row)
def __call__(self):
self.setup_data()
return self.template()
class Article11Compare(Article11):
template = Template('pt/report-data-compare.pt')
is_side_by_side = True
def __init__(self,
context,
request,
country_code,
region_code,
descriptor,
article,
muids,
data_2020,
filenames=None):
super(Article11Compare,
self).__init__(context, request, country_code, region_code,
descriptor, article, muids, filenames)
self.data_2020 = data_2020
def auto_translate(self):
try:
self.setup_data()
except AssertionError:
return
translatables = self.context.TRANSLATABLES
seen = set()
# initiate translation for 2014 data
for row in self.rows:
if not row:
continue
if row.title not in translatables:
continue
for value in row.raw_values:
if not isinstance(value, basestring):
continue
if value not in seen:
retrieve_translation(self.country_code, value)
seen.add(value)
# initiate translation for 2020 data
for row in self.data_2020[0][1]:
field = row[0]
if field.title not in translatables:
continue
values = row[1]
for value in values:
if not isinstance(value, basestring):
continue
if value not in seen:
retrieve_translation(self.country_code, value)
seen.add(value)
return ''
def __call__(self):
self.setup_data()
return self.template(data=self.rows, data_2020=self.data_2020)
class Article8ESA(BaseArticle2012):
# TODO not implemented, copy of Article 8
""" Article 8 implementation for nation descriptors data
klass(self, self.request, self.country_code, self.descriptor,
self.article, self.muids, self.colspan)
"""
template = Template('pt/report-data-a8.pt')
help_text = ""
def setup_data(self):
filename = self.context.get_report_filename()
text = get_xml_report_data(filename)
root = fromstring(text)
def xp(xpath, node=root):
return node.xpath(xpath, namespaces=NSMAP)
# TODO: should use declared set of marine unit ids
xml_muids = sorted(set(xp('//w:MarineUnitID/text()')))
self.rows = [
Row('Reporting area(s) [MarineUnitID]',
[', '.join(set(xml_muids))]),
]
report_map = defaultdict(list)
root_tags = get_report_tags(root)
ReportTag = None
# basic algorthim to detect what type of report it is
article = self.article
# override the default translatable
fields = REPORT_DEFS[self.context.year][article]\
.get_translatable_fields()
self.context.TRANSLATABLES.extend(fields)
for name in root_tags:
nodes = xp('//w:' + name)
for node in nodes:
try:
rep = ReportTag(node, NSMAP)
except:
# There are some cases when an empty node is reported
# and the ReportTag class cannot be initialized because
# MarineUnitID element is not present in the node
# see ../fi/bal/d5/art8/@@view-report-data-2012
# search for node MicrobialPathogens
continue
import pdb
pdb.set_trace()
# TODO for D7(maybe for other descriptors too)
# find a way to match the node with the descriptor
# because all reported criterias and indicators are GESOther
if rep.matches_descriptor(self.descriptor):
report_map[rep.marine_unit_id].append(rep)
descriptor = get_descriptor(self.descriptor)
ges_crits = [descriptor] + list(descriptor.criterions)
# a bit confusing code, we have multiple sets of rows, grouped in
# report_data under the marine unit id key.
report_data = {}
# TODO: use reported list of muids per country,from database
for muid in xml_muids:
if muid not in report_map:
logger.warning("MarineUnitID not reported: %s, %s, Article 8",
muid, self.descriptor)
report_data[muid] = []
continue
m_reps = report_map[muid]
if len(m_reps) > 1:
logger.warning(
"Multiple report tags for this "
"marine unit id: %r", m_reps)
rows = []
for i, report in enumerate(m_reps):
# if i > 0: # add a splitter row, to separate reports
# rows.append(Row('', ''))
cols = report.columns(ges_crits)
for col in cols:
for name in col.keys():
values = []
for inner in cols:
values.append(inner[name])
translated_values = [
self.context.translate_value(
name, v, self.country_code) for v in values
]
row = RawRow(name, translated_values, values)
rows.append(row)
break # only need the "first" row, for headers
report_data[muid] = rows
res = {}
muids = {m.id: m for m in self.muids}
for mid, v in report_data.items():
mlabel = muids.get(mid)
if mlabel is None:
logger.warning("Report for non-defined muids: %s", mid)
mid = unicode(mid)
mlabel = MarineReportingUnit(mid, mid)
res[mlabel] = v
# self.muids = sorted(res.keys())
self.rows = res
def __call__(self):
self.setup_data()
return self.template()
def auto_translate(self):
self.setup_data()
translatables = self.context.TRANSLATABLES
seen = set()
for table in self.rows.items():
muid, table_data = table
for row in table_data:
if not row:
continue
if row.title not in translatables:
continue
for value in row.raw_values:
if not isinstance(value, basestring):
continue
if value not in seen:
retrieve_translation(self.country_code, value)
seen.add(value)
return ''
class NationalDescriptorArticleView(BaseView, AssessmentDataMixin):
implements(INationaldescriptorArticleView)
section = 'national-descriptors'
assessment_data_2012_tpl = Template('./pt/assessment-data-2012.pt')
assessment_data_2018_tpl = Template('./pt/assessment-data-2018.pt')
year = '2018' # used by self.muids
_questions = NAT_DESC_QUESTIONS
@property
def title(self):
return u"Commission assessment / {} / 2018 / {} / {} / {} ".format(
self.article,
self.descriptor_title,
self.country_title,
self.country_region_name,
)
@property
def criterias(self):
return self.descriptor_obj.sorted_criterions() # criterions
@property
def questions(self):
qs = self._questions.get(self.article, [])
return qs
def __call__(self):
if 'assessor' in self.request.form:
assessors = self.request.form['assessor']
if isinstance(assessors, list):
assessors = ', '.join(assessors)
self.context.saved_assessment_data.ass_new = assessors
# BBB:
context = self.context
if not hasattr(context, 'saved_assessment_data') or \
not isinstance(context.saved_assessment_data, PersistentList):
context.saved_assessment_data = AssessmentData()
# Assessment data 2012
descriptor_criterions = get_descriptor(self.descriptor).criterions
country_name = self._country_folder.title
try:
db_data_2012 = get_assessment_data_2012_db(country_name,
self.descriptor,
self.article)
assessments_2012 = filter_assessment_data_2012(
db_data_2012,
self.country_region_code, # TODO: this will need refactor
descriptor_criterions,
)
self.assessment_data_2012 = self.assessment_data_2012_tpl(
data=assessments_2012)
if assessments_2012.get(country_name):
score_2012 = assessments_2012[country_name].score
conclusion_2012 = assessments_2012[country_name].overall_ass
else: # fallback
ctry = assessments_2012.keys()[0]
score_2012 = assessments_2012[ctry].score
conclusion_2012 = assessments_2012[ctry].overall_ass
report_by, assessors, assess_date, source_file = \
get_assessment_head_data_2012(self.article,
self.country_region_code,
self._country_folder.id)
except:
logger.exception("Could not get assessment data for 2012")
self.assessment_data_2012 = ''
score_2012 = 100
conclusion_2012 = 'Not found'
report_by, assessors, assess_date, source_file = [
'Not found'
] * 3 + [('Not found', '')]
# Assessment header 2012
self.assessment_header_2012 = self.assessment_header_template(
report_by=report_by,
assessor_list=[],
assessors=assessors,
assess_date=assess_date,
source_file=source_file,
show_edit_assessors=False,
)
# Assessment data 2018
data = self.context.saved_assessment_data.last()
elements = self.questions[0].get_all_assessed_elements(
self.descriptor_obj, muids=self.muids)
article_weights = ARTICLE_WEIGHTS
assessment = format_assessment_data(self.article, elements,
self.questions, self.muids, data,
self.descriptor_obj,
article_weights, self)
assessment.phase_overall_scores.coherence = self.get_coherence_data(
self.country_region_code, self.descriptor, self.article)
score_2012 = int(round(score_2012))
conclusion_2012_color = CONCLUSION_COLOR_TABLE.get(score_2012, 0)
change = int(
assessment.phase_overall_scores.get_range_index_for_phase(
'adequacy') - score_2012)
self.assessment_data_2018_html = self.assessment_data_2018_tpl(
assessment=assessment,
score_2012=score_2012,
conclusion_2012=conclusion_2012,
conclusion_2012_color=conclusion_2012_color,
change_since_2012=change,
can_comment=self.can_comment)
# Assessment header 2018
report_by_2018 = u'Commission'
# assessors_2018 = self.context.saved_assessment_data.assessors
assessors_2018 = getattr(self.context.saved_assessment_data, 'ass_new',
'Not assessed')
assess_date_2018 = data.get('assess_date', u'Not assessed')
source_file_2018 = ('To be addedd...', '.')
can_edit = self.check_permission('wise.msfd: Edit Assessment')
show_edit_assessors = self.assessor_list and can_edit
self.assessment_header_2018_html = self.assessment_header_template(
report_by=report_by_2018,
assessor_list=self.assessor_list,
assessors=assessors_2018,
assess_date=assess_date_2018,
source_file=source_file_2018,
show_edit_assessors=show_edit_assessors,
)
return self.index()