def createMembers(theme, pages):
instances = getThemeInstances(theme, pages)
for instance in instances:
name = instance.get("n", "")
unique_f = getUniqueFeatures([instance], instances)
num_f = getFeatures([instance])
unique_l = getUniqueLanguages([instance], instances)
num_l = list(set(getLangs([instance])))
unique_t = getUniqueTechs([instance], instances)
num_t = list(set(getTechs([instance])))
unique_c = getUniqueConcepts([instance], instances)
num_c = list(set(getConcepts([instance])))
headline = remove_headline_markup(instance.get("headline", ""))
yield {
"name": name,
"headline": headline,
"features": num_f,
"ufeatures": unique_f,
"languages": num_l,
"ulanguages": unique_l,
"technologies": num_t,
"utechnologies": unique_t,
"concepts": num_c,
"uconcepts": unique_c,
}
def handleContribution(name, map):
repoUrl = repo.get(name, '<unresolved>')
if repoUrl == '<unresolved>': problems.append({'missing repo url' : name})
pageName = page['page']['page']['p'] + ':' + page['page']['page']['n']
map[name] = {
'101wiki' : wikiUrl.format(pageName),
'101repo' : repoUrl,
'headline': remove_headline_markup( page['page'].get('headline', '<unresolved>') )
}
def createTechnologies(theme, pages):
theme_pages = getThemeInstances(theme, pages)
technologies = list(set(getTechs(theme_pages)))
for tech in technologies:
rf = getRealTechnology(tech, pages)
contributions = getContributionsWithTechnology(tech, theme_pages)
headline = remove_headline_markup(rf.get("headline", ""))
contributions = getContributionNames(contributions)
resolved = bool(rf.get("resolved", ""))
yield {"name": tech, "headline": headline, "contributions": contributions, "resolved": resolved}
def findWikiEntry(val, namespace, title, repoDir, map):
for page in wiki['pages']:
if page['page']['page']['p'] == namespace and page['page']['page']['n'] == title.replace('_', ' '):
map[val] = {
'101wiki': wikiUrl.format(encodeForUrl(namespace,title)),
'headline': remove_headline_markup( page['page'].get('headline', '<unresolved>') )
}
handleRepoLink(repoDir,namespace, title,map[val])
return
map[val] = {'101wiki': '<unresolved>', 'headline': '<unresolved>'}
problems.append({'missingWikiPage': wikiUrl.format(encodeForUrl(namespace, title))})
def createConcepts(theme, pages):
theme_pages = getThemeInstances(theme, pages)
concepts = list(set(getConcepts(theme_pages)))
for concept in concepts:
rf = getRealConcept(concept, pages)
contributions = getContributionsWithConcept(concept, theme_pages)
headline = remove_headline_markup(rf.get("headline", ""))
contributions = getContributionNames(contributions)
resolved = bool(rf["page"].get("resolved", ""))
yield {"name": concept, "headline": headline, "contributions": contributions, "resolved": resolved}
def handleContribution(name, map):
repoUrl = repo.get(name, '<unresolved>')
if repoUrl == '<unresolved>': problems.append({'missing repo url': name})
pageName = page['page']['page']['p'] + ':' + page['page']['page']['n']
map[name] = {
'101wiki':
wikiUrl.format(pageName),
'101repo':
repoUrl,
'headline':
remove_headline_markup(page['page'].get('headline', '<unresolved>'))
}
def createFeatures(theme, pages):
theme_pages = getThemeInstances(theme, pages)
features = list(set(getFeatures(theme_pages)))
feature_names = features
for feature in feature_names:
rf = getRealFeature(feature, pages)
contributions = getContributionsWithFeature(feature, theme_pages)
headline = remove_headline_markup(rf["page"].get("headline", ""))
contributions = getContributionNames(contributions)
resolved = bool(rf.get("resolved", ""))
yield {"name": feature, "headline": headline, "contributions": contributions, "resolved": resolved}
def findWikiEntry(val, namespace, title, repoDir, map):
for page in wiki['pages']:
if page['page']['page']['p'] == namespace and page['page']['page'][
'n'] == title.replace('_', ' '):
map[val] = {
'101wiki':
wikiUrl.format(encodeForUrl(namespace, title)),
'headline':
remove_headline_markup(page['page'].get(
'headline', '<unresolved>'))
}
handleRepoLink(repoDir, namespace, title, map[val])
return
map[val] = {'101wiki': '<unresolved>', 'headline': '<unresolved>'}
problems.append(
{'missingWikiPage': wikiUrl.format(encodeForUrl(namespace, title))})
def createTechnologies(theme, pages):
theme_pages = getLanguageInstances(theme, pages)
technologies = getTechs(theme_pages)
for tech in technologies:
rf = getRealTechnology(tech, pages)
contributions = getContributionsWithTechnology(tech, theme_pages)
headline = remove_headline_markup(rf.get('headline', ''))
contributions = getContributionNames(contributions)
resolved = bool(rf.get('resolved', ''))
yield {
'name': tech,
'headline': headline,
'contributions': contributions,
'resolved': resolved
}
def createConcepts(theme, pages):
theme_pages = getLanguageInstances(theme, pages)
concepts = getConcepts(theme_pages)
for concept in concepts:
rf = getRealConcept(concept, pages)
contributions = getContributionsWithConcept(concept, theme_pages)
headline = remove_headline_markup(rf.get('headline', ''))
contributions = getContributionNames(contributions)
resolved = bool(rf.get('resolved', ''))
yield {
'name': concept,
'headline': headline,
'contributions': contributions,
'resolved': resolved
}
def createFeatures(theme, pages):
theme_pages = getLanguageInstances(theme, pages)
features = getFeatures(theme_pages)
feature_names = features
for feature in feature_names:
rf = getRealFeature(feature, pages)
contributions = getContributionsWithFeature(feature, theme_pages)
headline = remove_headline_markup(rf.get('headline', ''))
contributions = getContributionNames(contributions)
resolved = bool(rf.get('resolved', ''))
yield {
'name': feature,
'headline': headline,
'contributions': contributions,
'resolved': resolved
}
def createMembers(theme, pages):
instances = getLanguageInstances(theme, pages)
for instance in instances:
name = instance.get('n', '')
unique_f = getUniqueFeatures([instance], instances)
num_f = getFeatures([instance])
unique_l = getUniqueLanguages([instance], instances)
num_l = list(set(getLangs([instance])))
unique_t = getUniqueTechs([instance], instances)
num_t = list(set(getTechs([instance])))
unique_c = getUniqueConcepts([instance], instances)
num_c = list(set(getConcepts([instance])))
headline = remove_headline_markup(instance.get('headline', ''))
yield {
'name': name,
'headline': headline,
'features': num_f,
'ufeatures': unique_f,
'Languages': num_l,
'uLanguages': unique_l,
'technologies': num_t,
'utechnologies': unique_t,
'concepts': num_c,
'uconcepts': unique_c
}
for voc in vocs:
voc_name = unicode('instanceOf::Vocabulary:' + voc).strip()
instances = (query(pages).where(lambda page: voc_name in page.get('internal_links', [])).to_list())
if not os.path.exists(os.path.join(output, voc)):
os.mkdir(os.path.join(output, voc))
f = open(os.path.join(output, voc, 'members.json'), 'w')
data = []
for instance in instances:
data.append({
'name': instance['n'],
#'namespace': instance['page']['page']['p'] or '',
'headline': remove_headline_markup(instance.get('headline', ''))
})
data = sorted(data, key=lambda s: s['name'])
for d in data:
if data.count(d) > 1:
data.remove(d)
json.dump(data, f, indent=4, sort_keys=True)
f.close()
loader = FileSystemLoader('.')
env = Environment(loader=loader)
template = env.get_template('html.tpl')
f = open(os.path.join(output, voc, 'members.html'), 'w')
def getHeadline(self, namespace, title):
page = self.selectPage(namespace, title)
if page:
return mediawiki.remove_headline_markup(page.get('headline', ''))
return None
def getHeadline(self, namespace, title):
page = self.selectPage(namespace, title)
if page:
return mediawiki.remove_headline_markup(page.get('headline', ''))
return None
def extractHeadline(namespace, pagetitle):
wikidata = json.load(open(const101.wikiDump, 'r'))
target = {'p': namespace, 'n': pagetitle}
for page in wikidata['wiki']['pages']:
if target == page['page'].get('page', None):
return remove_headline_markup(page['page']['headline'])
def extractHeadline(namespace, pagetitle):
wikidata = json.load(open(const101.wikiDump, 'r'))
target = {'p': namespace, 'n': pagetitle}
for page in wikidata['wiki']['pages']:
if target == page['page'].get('page', None):
return remove_headline_markup(page['page']['headline'])
