def test_TxtCitation(self): from clld.web.adapters import TxtCitation adapter = TxtCitation(None) self.assertTrue( '.' in adapter.render(Contribution.first(), self.env['request'])) adapter.render(Dataset.first(), self.env['request'])
def test_BibTex(self): from clld.web.adapters import BibTex adapter = BibTex(None) self.assertTrue( '@' in adapter.render(Contribution.first(), self.env['request']))
def justifications(args, languages): """ - text goes into ValueSet.description - refs go into ValueSetReference objects """ def normalized_pages(s): if PAGES_PATTERN.match(s or ""): return s or "" # # create mappings to look up glottolog languoids matching names in justification files # langs_by_hid = languages langs_by_hname = {} langs_by_name = {} for l in DBSession.query(Languoid).filter(Languoid.active == False): langs_by_hname[l.jsondatadict.get("hname")] = l langs_by_hid[l.hid] = l langs_by_name[l.name] = l for l in DBSession.query(Languoid).filter(Languoid.active == True): langs_by_hname[l.jsondatadict.get("hname")] = l langs_by_hid[l.hid] = l langs_by_name[l.name] = l for id_, type_ in [("fc", "family"), ("sc", "subclassification")]: for i, row in enumerate(dsv.reader(args.data_file("%s_justifications.tab" % type_))): name = row[0] name = name.replace("_", " ") if not name.startswith("NOCODE") else name l = langs_by_hname.get(name, langs_by_hid.get(name, langs_by_name.get(name))) if not l: args.log.warn("ignoring %s" % name) continue _r = 3 if type_ == "family" else 2 comment = (row[_r].strip() or None) if len(row) > _r else None if comment and not WORD_PATTERN.search(comment): comment = None # # TODO: look for [NOCODE_ppp] patterns as well!? # refs = [(int(m.group("id")), normalized_pages(m.group("comment"))) for m in REF_PATTERN.finditer(row[2])] vs = None for _vs in l.valuesets: if _vs.parameter.id == id_: vs = _vs break if not vs: args.log.info("%s %s ++" % (l.id, type_)) vs = ValueSet( id="%s%s" % (type_, l.id), description=comment, language=l, parameter=Parameter.get(id_), contribution=Contribution.first(), ) DBSession.add(Value(id="%s%s" % (type_, l.id), name="%s - %s" % (l.level, l.status), valueset=vs)) DBSession.flush() else: if vs.description != comment: args.log.info("%s %s ~~ description" % (l.id, type_)) vs.description = comment for r in vs.references: DBSession.delete(r) for r, pages in refs: vs.references.append(ValueSetReference(source=Source.get(str(r)), description=pages)) args.log.info("%s %s" % (i, type_))
def justifications(args, languages, stats): """ - text goes into ValueSet.description - refs go into ValueSetReference objects """ hh_bibkey_to_glottolog_id = {} for rec in get_bib(args): for provider, bibkeys in get_bibkeys(rec).items(): if provider == 'hh': for bibkey in bibkeys: hh_bibkey_to_glottolog_id[bibkey] = rec['glottolog_ref_id'] break def substitute_hh_bibkeys(m): return '**%s**' % hh_bibkey_to_glottolog_id[m.group('bibkey')] # # create mappings to look up glottolog languoids matching names in justification files # langs_by_hid = languages langs_by_hname = {} langs_by_name = {} # order by active to make sure, we active languoid overwrite the data of obsolete ones. for l in DBSession.query(Languoid).order_by(Languoid.active): langs_by_hname[l.jsondata.get('hname')] = l langs_by_hid[l.hid] = l langs_by_name[l.name] = l def normalize_pages(s): return (s or '').strip().rstrip(',') or None for id_, type_ in [('fc', 'family'), ('sc', 'subclassification')]: for i, row in enumerate(dsv.reader( args.data_dir.joinpath('languoids', 'forkel_%s_justifications-utf8.tab' % type_))): name = row[0] name = name.replace('_', ' ') if not name.startswith('NOCODE') else name l = langs_by_hname.get(name, langs_by_hid.get(name, langs_by_name.get(name))) if not l: args.log.warn('ignoring %s' % name) continue _r = 3 if type_ == 'family' else 2 comment = (row[_r].strip() or None) if len(row) > _r else None if comment and not WORD_PATTERN.search(comment): comment = None if comment: comment = re.sub('\*\*(?P<bibkey>[^\*]+)\*\*', substitute_hh_bibkeys, comment) # # TODO: look for [NOCODE_ppp] patterns as well!? # refs = [(int(m.group('id')), normalize_pages(m.group('pages'))) for m in REF_PATTERN.finditer( re.sub('\*\*(?P<bibkey>[^\*]+)\*\*', substitute_hh_bibkeys, row[2]))] vs = None for _vs in l.valuesets: if _vs.parameter.id == id_: vs = _vs break if not vs: args.log.info('%s %s ++' % (l.id, type_)) vs = ValueSet( id='%s%s' % (id_, l.pk), description=comment, language=l, parameter=Parameter.get(id_), contribution=Contribution.first()) DBSession.add(Value( id='%s%s' % (id_, l.pk), name='%s - %s' % (l.level, l.status), valueset=vs)) DBSession.flush() else: if vs.description != comment: args.log.info('%s %s ~~ description: %s ---> %s' % (l.id, type_, vs.description, comment)) vs.description = comment stats.update(['justifications-%s' % type_]) for r in vs.references: DBSession.delete(r) for r, pages in refs: # FIXME: we must make sure not to link sources which will subsequently be # replaced! vs.references.append(ValueSetReference( source=Source.get(str(r)), description=pages)) args.log.info('%s %s' % (i, type_))
def test_BibTex(env): from clld.web.adapters import BibTex adapter = BibTex(None) assert '@' in adapter.render(Contribution.first(), env['request'])
def test_TxtCitation(env): from clld.web.adapters import TxtCitation adapter = TxtCitation(None) assert '.' in adapter.render(Contribution.first(), env['request']) adapter.render(Dataset.first(), env['request'])