def setup_data(self): t = sql.MSFD9Descriptor muids = {m.id: m for m in self.muids} count, res = db.get_all_records( t, t.MarineUnitID.in_(muids.keys()), ) by_muid = defaultdict(list) descriptor = get_descriptor(self.descriptor) ok_ges_ids = descriptor.all_ids() for desc_item in res: ges_id = criteria_from_gescomponent(desc_item.ReportingFeature) if ges_id not in ok_ges_ids: continue item = A9AlternateItem(desc_item) by_muid[desc_item.MarineUnitID].append(item) self.rows = {} for muid, cols in by_muid.items(): rows = [] if not cols: continue for name in cols[0].keys(): values = [c[name] for c in cols] row = Row(name, values) rows.append(row) self.rows[muids[muid]] = rows
def setup_data(self): descriptor = get_descriptor(self.descriptor) ok_ges_ids = set(descriptor.all_ids()) t = sql.MSFD10Target # muids = [x.id for x in self.muids] muids = {m.id: m for m in self.muids} count, res = db.get_all_records( t, t.MarineUnitID.in_(muids.keys()), t.Topic == 'EnvironmentalTarget', ) by_muid = defaultdict(list) for target_item in res: item = A10AlternateItem(target_item, ok_ges_ids) if item.needed: by_muid[target_item.MarineUnitID].append(item) self.rows = {} for muid, cols in by_muid.items(): rows = [] if not cols: continue for name in cols[0].keys(): values = [c[name] for c in cols] row = Row(name, values) rows.append(row) self.rows[muids[muid]] = rows
def setup_data(self): # setup data to be available at 'features_pressures_data' attr self.get_features_pressures_data() data = self.features_pressures_data out = [] general_pressures = set([ 'PresAll', ]) # 'Unknown' for theme, features_for_theme in self.features_needed: theme_data = [] theme_general_pressures = set( [x for x in features_for_theme if x.endswith('All')]) for feature in features_for_theme: if feature.endswith('All'): continue feature_data = [] for country_id, country_name in self.available_countries: # if pressure is ending with 'All' it applies to all # features in the current theme descriptors_rep = [] for x in data: descr = x[1] try: descr_obj = get_descriptor(descr) descr = descr_obj.template_vars['title'] descr_name = descr_obj.name except: descr_name = descr ccode = x[0] if ccode != country_id: continue feats = set(x[2].split(',')) if (feature in feats or general_pressures.intersection(feats) or theme_general_pressures.intersection(feats)): descriptors_rep.append((descr_name, descr)) descriptors_rep = set(descriptors_rep) descriptors_rep = [ItemLabel(*x) for x in descriptors_rep] feature_data.append(ItemListOverview(descriptors_rep)) theme_data.append( (self.get_feature_short_name(feature), feature_data)) out.append((theme, theme_data)) return out
def all_descriptor_ids(self): descr_class = get_descriptor(self.descriptor) all_ids = list(descr_class.all_ids()) if self.descriptor.startswith('D1.'): all_ids.append('D1') all_ids = set(all_ids) return all_ids
def get_data_from_view_Art9(self): t = sql2018.t_V_ART9_GES_2018 descriptor = get_descriptor(self.descriptor) all_ids = list(descriptor.all_ids()) if self.descriptor.startswith('D1.'): all_ids.append('D1') conditions = [ t.c.CountryCode == self.country_code, t.c.GESComponent.in_(all_ids) ] if self.country_code != 'DK': conditions.insert( 1, or_(t.c.Region == self.country_region_code, t.c.Region.is_(None))) else: # Handle the case of Denmark that have submitted a lot of # information under the DK-TOTAL MRU, which doesn't have a region # attached. conditions.insert( 1, or_(t.c.Region == 'NotReported', t.c.Region == self.country_region_code, t.c.Region.is_(None))) count, q = db.get_all_records_ordered(t, ('GESComponent', ), *conditions) ok_features = set([f.name for f in get_features(self.descriptor)]) out = [] # There are cases when justification for delay is reported # for a ges component. In these cases region, mru, features and # other fields are empty. Justification for delay should be showed # for all regions, mrus for row in q: if not row.Features: out.append(row) continue if not self.descriptor.startswith('D1.'): out.append(row) continue feats = set(row.Features.split(',')) if feats.intersection(ok_features): out.append(row) return out
def insert_missing_criterions(data, descriptor): """ For Art9 we want to show a row for all possible GESComponents, regardless if the MS has reported on that or not :param data: a map of <muid>: list of rows :param descriptor: a Descriptor instance This function will change in place the provided data """ criterions = [] # need to add D1 descriptor to the criterion lists too, CY reported data # on the generic D1 descriptor if 'D1.' in descriptor.id: criterions.append(get_descriptor('D1')) criterions.append(descriptor) criterions.extend(descriptor.sorted_criterions()) for muidlist, dataset in data.items(): # build a map of existing criterions, so that we detect missing ones # this proxy object will serve as template for new cloned columns, # to be able to show an empty column tplobj = None # colmap = {} colmap = defaultdict(list) new = [] for col in dataset: # rewrite the GESComponent feature. TODO: move this functionality # to the Proxy2018 and XML file, with a getter # if col.GESComponent in colmap: # continue if col.GESComponent.is_descriptor(): colmap[col.GESComponent].append(col) else: col.GESComponent = descriptor[col.GESComponent.id] colmap[col.GESComponent].append(col) if tplobj is None: tplobj = col for c in criterions: if c in colmap: col = colmap[c] else: col = [tplobj.clone(GESComponent=c)] new.extend(col) data[muidlist] = new
def descriptors(self): descriptors = get_all_descriptors() descriptors.pop(0) # remove D1 general descriptor res = [] for desc in descriptors: desc_obj = get_descriptor(desc[0]) res.append((desc_obj.template_vars['title'], desc_obj.title)) return res
def filtered_ges_components(self, seed): """ Returns a list of valid ges criterion indicator targets Can be something like "1.6.2-indicator 5.2B" or "3.1" or "D1" """ descriptor = get_descriptor(self.descriptor) country_criterions = country_ges_components(self.country_code) res = set([self.descriptor]) for d_id in descriptor.all_ids(): if d_id in country_criterions: res.add(d_id) for crit in set(country_criterions + seed): crit_id = crit.split('-', 1)[0] if crit_id in descriptor.all_ids(): res.add(crit) return sorted_by_criterion(res)
def _art_4_ids(self, descriptor, **kwargs): """ Return all descriptors """ descriptors = get_all_descriptors() res = [] for descriptor in descriptors: descr_id = descriptor[0] if descr_id == 'D1': continue descr_title = descriptor[1] descriptor_obj = get_descriptor(descr_id) alternate_id = descriptor_obj.template_vars['title'] descr_opt = DescriptorOption(id=alternate_id, title=descr_title, is_primary=lambda _: True) res.append(descr_opt) return res
def get_criterias_list(self, descriptor): """ Get the list of criterias for the specified descriptor :param descriptor: 'D5' :return: (('D5', 'Eutrophication'), ('5.1.1', 'D5C1'), ('5.2.1', 'D5C2'), ... ) # TODO: the results here need to be augumented by L_GESComponents """ result = [(descriptor, self.descriptor_label)] criterions = get_descriptor(descriptor).criterions for crit in criterions: for alt in crit.alternatives: title = '{} ({}) {}'.format(crit._id or '', alt[0], alt[1]) indicator = alt[0] result.append((indicator, title)) return result
def get_data_from_view_Art8(self): sess = db.session() t = sql2018.t_V_ART8_GES_2018 descr_class = get_descriptor(self.descriptor) all_ids = list(descr_class.all_ids()) if self.descriptor.startswith('D1.'): all_ids.append('D1') # muids = [x.id for x in self.muids] conditions = [ t.c.CountryCode == self.country_code, # t.c.Region == self.country_region_code, # t.c.MarineReportingUnit.in_(muids), # t.c.GESComponent.in_(all_ids) ] # Handle the case of Romania that submitted duplicate data, # where Element is empty, but Criteria has data if self.country_code != 'RO': conditions.append( or_(t.c.Element.isnot(None), t.c.Criteria.isnot(None))) else: conditions.append(t.c.Element.isnot(None)) if self.country_code != 'DK': conditions.insert(1, t.c.Region == self.country_region_code) else: # Handle the case of Denmark that have submitted a lot of # information under the DK-TOTAL MRU, which doesn't have a region # attached. conditions.insert( 1, or_(t.c.Region == 'NotReported', t.c.Region == self.country_region_code)) orderby = [ getattr(t.c, x) for x in self._get_order_cols_Art8(self.descriptor) ] # groupby IndicatorCode q = sess\ .query(t)\ .filter(*conditions)\ .order_by(*orderby)\ .distinct() # For the following countries filter data by features # for other countries return all data country_filters = ('BE', ) if self.country_code not in country_filters: return q ok_features = set([f.name for f in get_features(self.descriptor)]) out = [] for row in q: if not self.descriptor.startswith('D1.'): out.append(row) continue feats = set((row.Feature, )) if feats.intersection(ok_features): out.append(row) return out
def descriptor_for_code(self, code): desc = get_descriptor(code.upper()) return desc
def get_data_old(self): mapper_class = sql2018.MRUsPublication mc_mru_descr = sql2018.MarineReportingUnit res = [] marine_waters_data = self.context._get_marine_waters_data() marine_waters_total = sum( [x[2] for x in marine_waters_data if x[0] == self.country_code]) # for better query speed we get only these columns col_names = ('Country', 'rZoneId', 'thematicId', 'nameTxtInt', 'nameText', 'spZoneType', 'legisSName', 'Area') columns = [getattr(mapper_class, name) for name in col_names] count, data = db.get_all_specific_columns( columns, mapper_class.Country == self.country_code) mrus_needed = [x.thematicId for x in data] _, mru_descriptions = db.get_all_specific_columns([ mc_mru_descr.MarineReportingUnitId, mc_mru_descr.Description, mc_mru_descr.nameTxtInt, mc_mru_descr.nameText ], mc_mru_descr.MarineReportingUnitId.in_(mrus_needed)) _, art8_data = db.get_all_specific_columns([ sql2018.t_V_ART8_GES_2018.c.MarineReportingUnit, sql2018.t_V_ART8_GES_2018.c.GESComponent ], sql2018.t_V_ART8_GES_2018.c.CountryCode == self.country_code) _, art9_data = db.get_all_specific_columns([ sql2018.t_V_ART9_GES_2018.c.MarineReportingUnit, sql2018.t_V_ART9_GES_2018.c.GESComponent ], sql2018.t_V_ART9_GES_2018.c.CountryCode == self.country_code) art8_art9_data = set(art8_data + art9_data) for row in data: mru = row.thematicId description = [ x.Description # (x.nameTxtInt is not None and x.nameTxtInt.strip() # or x.nameText or "") for x in mru_descriptions if x.MarineReportingUnitId == mru ] if not description: description = row.nameTxtInt or row.nameText or "" else: description = description[0] translation = get_translated(description, self.country_code) or "" area = int(round(row.Area)) if not translation: retrieve_translation(self.country_code, description) self._translatable_values.append(description) prop_water = int(round((area / marine_waters_total) * 100)) descr_list = set( [x[1] for x in art8_art9_data if x[0] == row.thematicId]) descr_list = sorted(descr_list, key=natural_sort_key) descr_list_norm = [] for d in descr_list: try: desc = get_descriptor(d).template_vars['title'] except: desc = d descr_list_norm.append(desc) descriptors = ', '.join(descr_list_norm) res.append( (row.rZoneId, row.spZoneType, mru, description, translation, '{:,}'.format(area), prop_water, descriptors)) return res
def __call__(self): alsoProvides(self.request, IDisableCSRFProtection) if 'assessor' in self.request.form: assessors = self.request.form['assessor'] if isinstance(assessors, list): assessors = ', '.join(assessors) self.context.saved_assessment_data.ass_new = assessors # BBB: context = self.context if not hasattr(context, 'saved_assessment_data') or \ not isinstance(context.saved_assessment_data, PersistentList): context.saved_assessment_data = AssessmentData() # Assessment data 2012 descriptor_criterions = get_descriptor(self.descriptor).criterions country_name = self._country_folder.title try: db_data_2012 = get_assessment_data_2012_db(country_name, self.descriptor, self.article) assessments_2012 = filter_assessment_data_2012( db_data_2012, self.country_region_code, # TODO: this will need refactor descriptor_criterions, ) self.assessment_data_2012 = self.assessment_data_2012_tpl( data=assessments_2012) if assessments_2012.get(country_name): score_2012 = assessments_2012[country_name].score conclusion_2012 = assessments_2012[country_name].overall_ass else: # fallback ctry = assessments_2012.keys()[0] score_2012 = assessments_2012[ctry].score conclusion_2012 = assessments_2012[ctry].overall_ass report_by, assessors, assess_date, source_file = \ get_assessment_head_data_2012(self.article, self.country_region_code, self._country_folder.id) except: logger.exception("Could not get assessment data for 2012") self.assessment_data_2012 = '' score_2012 = 0 conclusion_2012 = 'Not found' report_by, assessors, assess_date, source_file = [ 'Not found' ] * 3 + [('Not found', '')] # Assessment header 2012 self.assessment_header_2012 = self.assessment_header_template( report_by=report_by, assessor_list=[], assessors=assessors, assess_date=assess_date, source_file=source_file, show_edit_assessors=False, show_file_version=False, ) # Assessment data 2018 data = self.context.saved_assessment_data.last() elements = self.questions[0].get_all_assessed_elements( self.descriptor_obj, muids=self.muids) article_weights = ARTICLE_WEIGHTS assessment = format_assessment_data(self.article, elements, self.questions, self.muids, data, self.descriptor_obj, article_weights, self) assessment.phase_overall_scores.coherence = self.get_coherence_data( self.country_region_code, self.descriptor, self.article) # score_2012 = score_2012 conclusion_2012_color = CONCLUSION_COLOR_TABLE.get(score_2012, 0) change = (assessment.phase_overall_scores.get_range_index_for_phase( 'adequacy') - score_2012) # if 2018 adequacy is not relevant, change since 2012 is not relevant if assessment.phase_overall_scores.adequacy['conclusion'][0] == '-': change = 'Not relevant (-)' self.assessment_data_2018_html = self.assessment_data_2018_tpl( assessment=assessment, score_2012=score_2012, conclusion_2012=conclusion_2012, conclusion_2012_color=conclusion_2012_color, change_since_2012=change, can_comment=self.can_comment) # Assessment header 2018 report_by_2018 = u'Commission' # assessors_2018 = self.context.saved_assessment_data.assessors assessors_2018 = getattr(self.context.saved_assessment_data, 'ass_new', 'Not assessed') assess_date_2018 = data.get('assess_date', u'Not assessed') source_file_2018 = ('To be addedd...', '.') can_edit = self.check_permission('wise.msfd: Edit Assessment') show_edit_assessors = self.assessor_list and can_edit file_version = self.get_file_version(self.country_date_assessed) self.assessment_header_2018_html = self.assessment_header_template( report_by=report_by_2018, assessor_list=self.assessor_list, assessors=assessors_2018, assess_date=assess_date_2018, source_file=source_file_2018, show_edit_assessors=show_edit_assessors, show_file_version=True, file_version=file_version) return self.index()
def setup_data(self): self.article9_cols = self.get_article9_columns() filename = self.context.get_report_filename() if not isinstance(filename, tuple): filename = [filename] _cols = [] _muids = [] for fname in filename: text = get_xml_report_data(fname) if not text: self.rows = [] return self.template() root = fromstring(text) def xp(xpath, node=root): return node.xpath(xpath, namespaces=NSMAP) muids = xp('//w:MarineUnitID/text()') count, res = db.get_marine_unit_id_names(list(set(muids))) labels = [ItemLabel(m, u'{} ({})'.format(t, m)) for m, t in res] # special case for PL where marine_unit_ids are not imported into DB # therefore we cannot get the labels for them if muids and not labels: labels = [ItemLabel(m, m) for m in set(muids)] _muids.extend(labels) muids = ItemList(labels) descriptor = get_descriptor(self.descriptor) self.descriptor_label = descriptor.title reported = xp("//w:DesriptorCriterionIndicator/text()") gcs = self.filtered_ges_components(reported) self.rows = [] # wrap the target per MarineUnitID all_target_indicators = [ TargetsIndicators(node) for node in xp('w:TargetsIndicators') ] cols = [ A10Item(self, gc, all_target_indicators, self.country_code, self.region_code, muids) for gc in gcs ] _cols.extend(cols) self.muids_labeled = sorted(_muids, key=lambda l: natural_sort_key(l.name)) self.cols = _cols # unwrap the columns into rows for col in _cols: for name in col.keys(): values = [] for inner in _cols: values.append(inner[name]) raw_values = [] vals = [] for v in values: raw_values.append(v) vals.append( self.context.translate_value(name, v, self.country_code)) row = RawRow(name, vals, raw_values) self.rows.append(row) break # only need the "first" row
def get_data(self): mapper_class = sql2018.MRUsPublication mc_mru_descr = sql2018.MarineReportingUnit data = _4GEO_DATA data_filtered = [ row for row in data if row.MemberState == self.country_code ] # for better query speed we get only these columns col_names = ('Country', 'rZoneId', 'thematicId', 'nameTxtInt', 'nameText', 'spZoneType', 'legisSName', 'Area') columns = [getattr(mapper_class, name) for name in col_names] _, data_db = db.get_all_specific_columns( columns, mapper_class.Country == self.country_code) res = [] _, art8_data = db.get_all_specific_columns([ sql2018.t_V_ART8_GES_2018.c.MarineReportingUnit, sql2018.t_V_ART8_GES_2018.c.GESComponent ], sql2018.t_V_ART8_GES_2018.c.CountryCode == self.country_code) _, art9_data = db.get_all_specific_columns([ sql2018.t_V_ART9_GES_2018.c.MarineReportingUnit, sql2018.t_V_ART9_GES_2018.c.GESComponent ], sql2018.t_V_ART9_GES_2018.c.CountryCode == self.country_code) art8_art9_data = set(art8_data + art9_data) for row in data_filtered: # if row.Status == 'Not used': # continue description = row.MRUName mru_id = self.norm_mru_id(row.MRUID) translation = get_translated(description, self.country_code) or '' area = row.MRUArea and int(round(row.MRUArea)) or 0 if not area: area = [x.Area for x in data_db if x.thematicId == mru_id] area = area and int(round(area[0])) or 0 if not translation: retrieve_translation(self.country_code, description) self._translatable_values.append(description) # prop_water = int(round((area / marine_waters_total) * 100)) prop_water = (row.MRUCoverage and "{0:.1f}".format(float(row.MRUCoverage)) or 0) if not prop_water: mw_total = [ x.MarineWatersArea for x in data_filtered if x.MarineWatersArea ] mw_total = mw_total and mw_total[0] or 0 prop_water = int(round((area / mw_total) * 100)) descr_list = set([x[1] for x in art8_art9_data if x[0] == mru_id]) descr_list = sorted(descr_list, key=natural_sort_key) descr_list_norm = [] for d in descr_list: try: desc = get_descriptor(d).template_vars['title'] except: desc = d descr_list_norm.append(desc) descriptors = ', '.join(descr_list_norm) res.append( (row.MarineRegion, row.AssessmentArea, mru_id, description, translation, '{:,}'.format(area), prop_water, descriptors)) return res
def setup_data(self): descriptor_class = get_descriptor(self.descriptor) all_ids = descriptor_class.all_ids() self.descriptor_label = descriptor_class.title if self.descriptor.startswith('D1.'): all_ids.add('D1') fileurls = self._filename _mp_nodes = [] _sub_prog_nodes = [] # separate Monitoring Programmes from Sub Programmes for fileurl in fileurls: try: root = self.get_report_file_root(fileurl) except XMLSyntaxError: continue region = xp('//Region', root) if region: region = region[0].text if region not in self.context.regions: continue if root.tag == 'MON': nodes = xp('//MonitoringProgrammes/*', root) _mp_nodes.extend(nodes) if root.tag == 'MONSub': nodes = xp('//SubProgrammes/*', root) _sub_prog_nodes.extend(nodes) # filter duplicate MP nodes, only keep the latest mp_seen = [] mp_nodes = [] for mp_node in _mp_nodes: if mp_node.tag in mp_seen: continue mp_nodes.append(mp_node) # filter duplicate SubProg nodes, only keep the latest sub_prog_seen = [] sub_prog_nodes = [] for sub_node in _sub_prog_nodes: subprog_id = xp('./Q4g_SubProgrammeID/text()', sub_node) if subprog_id in sub_prog_seen: continue sub_prog_nodes.append(sub_node) items = [] for mp in mp_nodes: # filter empty nodes if not mp.getchildren(): continue # filter mp node by ges criteria ges_crit = xp('.//Q5a_RelevantGESCriteria', mp) if ges_crit: ges_crit_text = ges_crit[0].text ges_crit = (ges_crit_text and set(ges_crit_text.split(' ')) or set()) if not all_ids.intersection(ges_crit): continue subprogrammes = xp('.//ReferenceSubProgramme', mp) for sub_prog in subprogrammes: subprog_id = xp('./SubMonitoringProgrammeID/text()', sub_prog) subprog_id = subprog_id[0].replace('.xml', '').strip() subp_name = xp('./SubMonitoringProgrammeName/text()', sub_prog) sub_prog_node = [ x for x in sub_prog_nodes if xp('./Q4g_SubProgrammeID', x)[0].text == subprog_id ] sub_prog_node = (len(sub_prog_node) and sub_prog_node[0] or SUBEMPTY) item = self._make_item(mp, sub_prog_node, subp_name[0]) items.append(item) self.rows = [] items = sorted(items, key=lambda i: [getattr(i, o) for o in self.sort_order]) self.cols = items self.items_to_rows(items)
def matches_descriptor(self, descriptor): descriptor = get_descriptor(descriptor) self_crits = set(self.indicators + self.criterias + self.criterias_from_topics) return bool(self_crits.intersection(descriptor.all_ids()))
def matches_descriptor(self, descriptor): descriptor = get_descriptor(descriptor) self_crits = set(self.criterias) return bool(self_crits.intersection(descriptor.all_ids()))
def setup_data(self, filename=None): if not filename: filename = self.context.get_report_filename() if not isinstance(filename, tuple): filename = [filename] def filter_descriptors(nodes, descriptor): res = [] for d in nodes: rf = x('w:ReportingFeature/text()', d)[0] rf = criteria_from_gescomponent(rf) if rf in all_ids: res.append(d) return res descriptor_class = get_descriptor(self.descriptor) all_ids = descriptor_class.all_ids() self.descriptor_label = descriptor_class.title if self.descriptor.startswith('D1.'): all_ids.add('D1') _cols = [] _muids = [] for fname in filename: text = get_xml_report_data(fname) if not text: self.rows = [] return self.template() root = fromstring(text) def x(xpath, node=root): return node.xpath(xpath, namespaces=NSMAP) # these are the records we are interested in descriptors = filter_descriptors(x('//w:Descriptors'), self.descriptor) # the xml file is a collection of records that need to be agregated # in order to "simplify" their information. For example, the # ThresholdValues need to be shown for all MarineUnitIds, but the # grouping criteria is the "GES Component" cols = [] seen = [] muids = root.xpath('//w:MarineUnitID/text()', namespaces=NSMAP) count, res = db.get_marine_unit_id_names(list(set(muids))) muid_ids = [y.id for y in self.muids] labels = [ItemLabel(m, t or m) for m, t in res if m in muid_ids] # special case for PL where marine_unit_ids are not imported into DB # therefore we cannot get the labels for them if muids and not labels: labels = [ItemLabel(m, m) for m in set(muids)] self.muids_labeled = sorted(labels, key=lambda l: natural_sort_key(l.name)) _muids.extend(labels) muids = ItemList(labels) for node in descriptors: col_id = node.find('w:ReportingFeature', namespaces=NSMAP).text if col_id in seen: continue item = A9Item(self, node, descriptors, muids) cols.append(item) seen.append(item.id) _cols.extend(cols) _muids = ItemList(_muids) # insert missing criterions self.insert_missing_criterions(descriptor_class, _cols, _muids) self.rows = [] sorted_ges_c = sorted_by_criterion([c.ges_component() for c in _cols]) def sort_func(col): return sorted_ges_c.index(col.ges_component()) sorted_cols = sorted(_cols, key=sort_func) self.cols = list(sorted_cols) for col in sorted_cols: for name in col.keys(): values = [] for inner in sorted_cols: values.append(inner[name]) raw_values = [] vals = [] for v in values: raw_values.append(v) vals.append( self.context.translate_value(name, v, self.country_code)) # values = [self.context.translate_value(name, value=v) # for v in values] row = RawRow(name, vals, raw_values) self.rows.append(row) break # only need the "first" row
def setup_data(self): filename = self.context.get_report_filename() text = get_xml_report_data(filename) root = fromstring(text) def xp(xpath, node=root): return node.xpath(xpath, namespaces=NSMAP) # TODO: should use declared set of marine unit ids xml_muids = sorted(set(xp('//w:MarineUnitID/text()'))) self.rows = [ Row('Reporting area(s) [MarineUnitID]', [', '.join(set(xml_muids))]), ] report_map = defaultdict(list) root_tags = get_report_tags(root) ReportTag = None # basic algorthim to detect what type of report it is article = self.article # override the default translatable fields = REPORT_DEFS[self.context.year][article]\ .get_translatable_fields() self.context.TRANSLATABLES.extend(fields) for name in root_tags: nodes = xp('//w:' + name) for node in nodes: try: rep = ReportTag(node, NSMAP) except: # There are some cases when an empty node is reported # and the ReportTag class cannot be initialized because # MarineUnitID element is not present in the node # see ../fi/bal/d5/art8/@@view-report-data-2012 # search for node MicrobialPathogens continue import pdb pdb.set_trace() # TODO for D7(maybe for other descriptors too) # find a way to match the node with the descriptor # because all reported criterias and indicators are GESOther if rep.matches_descriptor(self.descriptor): report_map[rep.marine_unit_id].append(rep) descriptor = get_descriptor(self.descriptor) ges_crits = [descriptor] + list(descriptor.criterions) # a bit confusing code, we have multiple sets of rows, grouped in # report_data under the marine unit id key. report_data = {} # TODO: use reported list of muids per country,from database for muid in xml_muids: if muid not in report_map: logger.warning("MarineUnitID not reported: %s, %s, Article 8", muid, self.descriptor) report_data[muid] = [] continue m_reps = report_map[muid] if len(m_reps) > 1: logger.warning( "Multiple report tags for this " "marine unit id: %r", m_reps) rows = [] for i, report in enumerate(m_reps): # if i > 0: # add a splitter row, to separate reports # rows.append(Row('', '')) cols = report.columns(ges_crits) for col in cols: for name in col.keys(): values = [] for inner in cols: values.append(inner[name]) translated_values = [ self.context.translate_value( name, v, self.country_code) for v in values ] row = RawRow(name, translated_values, values) rows.append(row) break # only need the "first" row, for headers report_data[muid] = rows res = {} muids = {m.id: m for m in self.muids} for mid, v in report_data.items(): mlabel = muids.get(mid) if mlabel is None: logger.warning("Report for non-defined muids: %s", mid) mid = unicode(mid) mlabel = MarineReportingUnit(mid, mid) res[mlabel] = v # self.muids = sorted(res.keys()) self.rows = res
def descriptor_obj(self): return get_descriptor(self.descriptor)
def descriptor_obj(self, descriptor): return get_descriptor(descriptor)