def createMembers(theme, pages): instances = getThemeInstances(theme, pages) for instance in instances: name = instance.get("n", "") unique_f = getUniqueFeatures([instance], instances) num_f = getFeatures([instance]) unique_l = getUniqueLanguages([instance], instances) num_l = list(set(getLangs([instance]))) unique_t = getUniqueTechs([instance], instances) num_t = list(set(getTechs([instance]))) unique_c = getUniqueConcepts([instance], instances) num_c = list(set(getConcepts([instance]))) headline = remove_headline_markup(instance.get("headline", "")) yield { "name": name, "headline": headline, "features": num_f, "ufeatures": unique_f, "languages": num_l, "ulanguages": unique_l, "technologies": num_t, "utechnologies": unique_t, "concepts": num_c, "uconcepts": unique_c, }
def handleContribution(name, map): repoUrl = repo.get(name, '<unresolved>') if repoUrl == '<unresolved>': problems.append({'missing repo url' : name}) pageName = page['page']['page']['p'] + ':' + page['page']['page']['n'] map[name] = { '101wiki' : wikiUrl.format(pageName), '101repo' : repoUrl, 'headline': remove_headline_markup( page['page'].get('headline', '<unresolved>') ) }
def createTechnologies(theme, pages): theme_pages = getThemeInstances(theme, pages) technologies = list(set(getTechs(theme_pages))) for tech in technologies: rf = getRealTechnology(tech, pages) contributions = getContributionsWithTechnology(tech, theme_pages) headline = remove_headline_markup(rf.get("headline", "")) contributions = getContributionNames(contributions) resolved = bool(rf.get("resolved", "")) yield {"name": tech, "headline": headline, "contributions": contributions, "resolved": resolved}
def findWikiEntry(val, namespace, title, repoDir, map): for page in wiki['pages']: if page['page']['page']['p'] == namespace and page['page']['page']['n'] == title.replace('_', ' '): map[val] = { '101wiki': wikiUrl.format(encodeForUrl(namespace,title)), 'headline': remove_headline_markup( page['page'].get('headline', '<unresolved>') ) } handleRepoLink(repoDir,namespace, title,map[val]) return map[val] = {'101wiki': '<unresolved>', 'headline': '<unresolved>'} problems.append({'missingWikiPage': wikiUrl.format(encodeForUrl(namespace, title))})
def createConcepts(theme, pages): theme_pages = getThemeInstances(theme, pages) concepts = list(set(getConcepts(theme_pages))) for concept in concepts: rf = getRealConcept(concept, pages) contributions = getContributionsWithConcept(concept, theme_pages) headline = remove_headline_markup(rf.get("headline", "")) contributions = getContributionNames(contributions) resolved = bool(rf["page"].get("resolved", "")) yield {"name": concept, "headline": headline, "contributions": contributions, "resolved": resolved}
def handleContribution(name, map): repoUrl = repo.get(name, '<unresolved>') if repoUrl == '<unresolved>': problems.append({'missing repo url': name}) pageName = page['page']['page']['p'] + ':' + page['page']['page']['n'] map[name] = { '101wiki': wikiUrl.format(pageName), '101repo': repoUrl, 'headline': remove_headline_markup(page['page'].get('headline', '<unresolved>')) }
def createFeatures(theme, pages): theme_pages = getThemeInstances(theme, pages) features = list(set(getFeatures(theme_pages))) feature_names = features for feature in feature_names: rf = getRealFeature(feature, pages) contributions = getContributionsWithFeature(feature, theme_pages) headline = remove_headline_markup(rf["page"].get("headline", "")) contributions = getContributionNames(contributions) resolved = bool(rf.get("resolved", "")) yield {"name": feature, "headline": headline, "contributions": contributions, "resolved": resolved}
def findWikiEntry(val, namespace, title, repoDir, map): for page in wiki['pages']: if page['page']['page']['p'] == namespace and page['page']['page'][ 'n'] == title.replace('_', ' '): map[val] = { '101wiki': wikiUrl.format(encodeForUrl(namespace, title)), 'headline': remove_headline_markup(page['page'].get( 'headline', '<unresolved>')) } handleRepoLink(repoDir, namespace, title, map[val]) return map[val] = {'101wiki': '<unresolved>', 'headline': '<unresolved>'} problems.append( {'missingWikiPage': wikiUrl.format(encodeForUrl(namespace, title))})
def createTechnologies(theme, pages): theme_pages = getLanguageInstances(theme, pages) technologies = getTechs(theme_pages) for tech in technologies: rf = getRealTechnology(tech, pages) contributions = getContributionsWithTechnology(tech, theme_pages) headline = remove_headline_markup(rf.get('headline', '')) contributions = getContributionNames(contributions) resolved = bool(rf.get('resolved', '')) yield { 'name': tech, 'headline': headline, 'contributions': contributions, 'resolved': resolved }
def createConcepts(theme, pages): theme_pages = getLanguageInstances(theme, pages) concepts = getConcepts(theme_pages) for concept in concepts: rf = getRealConcept(concept, pages) contributions = getContributionsWithConcept(concept, theme_pages) headline = remove_headline_markup(rf.get('headline', '')) contributions = getContributionNames(contributions) resolved = bool(rf.get('resolved', '')) yield { 'name': concept, 'headline': headline, 'contributions': contributions, 'resolved': resolved }
def createFeatures(theme, pages): theme_pages = getLanguageInstances(theme, pages) features = getFeatures(theme_pages) feature_names = features for feature in feature_names: rf = getRealFeature(feature, pages) contributions = getContributionsWithFeature(feature, theme_pages) headline = remove_headline_markup(rf.get('headline', '')) contributions = getContributionNames(contributions) resolved = bool(rf.get('resolved', '')) yield { 'name': feature, 'headline': headline, 'contributions': contributions, 'resolved': resolved }
def createMembers(theme, pages): instances = getLanguageInstances(theme, pages) for instance in instances: name = instance.get('n', '') unique_f = getUniqueFeatures([instance], instances) num_f = getFeatures([instance]) unique_l = getUniqueLanguages([instance], instances) num_l = list(set(getLangs([instance]))) unique_t = getUniqueTechs([instance], instances) num_t = list(set(getTechs([instance]))) unique_c = getUniqueConcepts([instance], instances) num_c = list(set(getConcepts([instance]))) headline = remove_headline_markup(instance.get('headline', '')) yield { 'name': name, 'headline': headline, 'features': num_f, 'ufeatures': unique_f, 'Languages': num_l, 'uLanguages': unique_l, 'technologies': num_t, 'utechnologies': unique_t, 'concepts': num_c, 'uconcepts': unique_c }
for voc in vocs: voc_name = unicode('instanceOf::Vocabulary:' + voc).strip() instances = (query(pages).where(lambda page: voc_name in page.get('internal_links', [])).to_list()) if not os.path.exists(os.path.join(output, voc)): os.mkdir(os.path.join(output, voc)) f = open(os.path.join(output, voc, 'members.json'), 'w') data = [] for instance in instances: data.append({ 'name': instance['n'], #'namespace': instance['page']['page']['p'] or '', 'headline': remove_headline_markup(instance.get('headline', '')) }) data = sorted(data, key=lambda s: s['name']) for d in data: if data.count(d) > 1: data.remove(d) json.dump(data, f, indent=4, sort_keys=True) f.close() loader = FileSystemLoader('.') env = Environment(loader=loader) template = env.get_template('html.tpl') f = open(os.path.join(output, voc, 'members.html'), 'w')
def getHeadline(self, namespace, title): page = self.selectPage(namespace, title) if page: return mediawiki.remove_headline_markup(page.get('headline', '')) return None
def extractHeadline(namespace, pagetitle): wikidata = json.load(open(const101.wikiDump, 'r')) target = {'p': namespace, 'n': pagetitle} for page in wikidata['wiki']['pages']: if target == page['page'].get('page', None): return remove_headline_markup(page['page']['headline'])