def download_lgw_content():
"""
:return:
"""
# parameters
base_url = 'https://libregamewiki.org'
destination_path = os.path.join(constants.root_path, 'code', 'lgw-import')
utils.recreate_directory(destination_path)
# read and process the base url (get all games and categories)
url = base_url + '/Category:Games'
games = []
while True:
text = requests.get(url).text
soup = BeautifulSoup(text, 'html.parser')
#categories = soup.find('div', id='mw-subcategories').find_all('li')
#categories = [(x.a['href'], x.a.string) for x in categories]
# game pages
pages = soup.find('div', id='mw-pages').find_all('li')
games.extend(((x.a['href'], x.a.string) for x in pages))
# next page
next_page = soup.find('a', string='next page')
if not next_page:
break
url = base_url + next_page['href']
# remove all those that start with user
games = [
game for game in games
if not any(game[1].startswith(x)
for x in ('User:'******'Template:', 'Bullet'))
]
print('current number of games in LGW {}'.format(len(games)))
for game in games:
print(game[1])
url = base_url + game[0]
destination_file = os.path.join(
destination_path,
osg.canonical_entry_name(game[0][1:]) + '.html')
text = requests.get(url).text
utils.write_text(destination_file, text)
index = divide_in_columns(developers_by_alphabet, developer_index)
index['title'] = 'Open source game developers'
index['subtitle'] = make_text('Alphabetical index of {} developers'.format(len(developers)))
index['categories'] = extended_alphabet
index['category-names'] = extended_alphabet_names
index['number_entries_per_category_threshold'] = 10
index['entry_bold'] = lambda x: 'tags' in x
index['category-infos'] = {}
write(template_categorical_index.render(index=index), developers_index_path)
if __name__ == "__main__":
# clean the output directory
print('clean current static website')
utils.recreate_directory(c.web_path)
# load entries, inspirations and developers and sort them
print('load entries, inspirations and developers')
entries = osg.read_entries()
entries.sort(key=lambda x: str.casefold(x['Title']))
inspirations = osg.read_inspirations()
inspirations = list(inspirations.values())
inspirations.sort(key=lambda x: str.casefold(x['Name']))
developers = osg.read_developers()
developers = list(developers.values())
developers.sort(key=lambda x: str.casefold(x['Name']))
# re-generate static website