def __init__(self):
#Options
text = """Starting from a list of Wikipedia categories written by the user in
'config.cfg' file, the script:
- downloads/updates a national OSM data file
- downloads from (from Quick Intersection) Wikipedia data regarding the selected
categories (subcategories and articles names)
- creates webpages for showing which articles are already tagged and
which ones are not.
"""
parser = argparse.ArgumentParser(description=text)
group = parser.add_mutually_exclusive_group()
#Manage OSM data
parser.add_argument("-d", "--download_osm",
help="Download OSM data of the country (from Geofabrik)",
action="store_true")
parser.add_argument("-u", "--update_osm",
help="Update downloaded OSM data of the country (through osmupdate)",
action="store_true")
#Analyze data from Wikipedia and OSM
parser.add_argument("-a", "--analyze",
help="Analyze Wikipedia data (categories' sub-categories and articles) ed OSM data (existing Wikipedia tags)",
action="store_true")
parser.add_argument("--category_info",
help="Analyze data and print informations regarding a specific category",
action="store")
parser.add_argument("-t", "--show_missing_templates",
help="Mark on web pages the articles that miss geo template (Coord)",
action="store_true")
parser.add_argument("-c", "--show_link_to_wikipedia_coordinates",
help="If a non-tagged article have the coordinates on Wikipedia, show on the web pages a link to zoom on its position with JOSM/iD",
action="store_true")
parser.add_argument("-o", "--show_coordinates_from_osm",
help="Calculate OSM coordinates of articles (point for nodes, centroids for ways and relations)",
action="store_true")
parser.add_argument("-n", "--infer_coordinates_from_wikipedia",
help="Use Nuts4Nuts to calculate the coordinates of a non tagged article whithout coordinates on Wikipedia",
action="store_true")
group.add_argument("-p", "--print_categories_list",
help="Analyze data and print project's categories.",
action="store_true")
#Create webpages
group.add_argument("-w", "--create_webpages",
help="Analyze data and create web pages",
action="store_true")
parser.add_argument("-s", "--save_stats",
help="If web pages have been created, store the updated number of tagged articles (default: ask to user).",
action="store_true")
parser.add_argument("--browser",
help="Open the web pages with the system browser after creation.",
action="store_true")
parser.add_argument("--copy",
help="Copy html folder to the directory configured on `config.cfg` (eg. dropbox dir).",
action="store_true")
parser.add_argument("--locale",
nargs='+',
dest='locales',
metavar='LANG',
help="Generate pages in the specified locales. Default: use the system locale. ")
self.args = parser.parse_args()
if self.args.category_info or self.args.category_info\
or self.args.create_webpages or self.args.print_categories_list\
or self.args.show_missing_templates\
or self.args.show_coordinates_from_osm:
self.args.analyze = True
# Default value for locale
# get system locale
sys_locale_langcode, sys_locale_encoding = locale.getdefaultlocale()
if not self.args.locales:
self.args.locales = [sys_locale_langcode]
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
os.chdir(os.path.dirname(sys.argv[0]))
#Configurations
themesAndCatsNames = self.read_config()
### Manage OpenStreetMap data ##########################################
#Analyse national OSM data file and create lists of already
#tagged Wikipedia articles.
#Download/update OSM data
if self.args.download_osm or self.args.update_osm:
if self.args.download_osm:
OSM.download_osm_data(self)
if self.args.update_osm:
status = OSM.update_osm_data(self)
if self.args.download_osm or (self.args.update_osm and status):
OSM.filter_wikipedia_data_in_osm_file(self)
if self.args.update_osm and not status:
print "OSM data where already uptodate or osmupdate has been interrupted.\
To repeat the updating process, launch the script again with the `-u` option."
if not self.args.analyze:
#"There's nothing left for me to tell you"
sys.exit(1)
else:
if not os.path.isfile(self.wOSMFile):
OSM.filter_wikipedia_data_in_osm_file(self)
#Extract Wikipedia articles tagged in OSM with preferred language.
#If an article is tagged in a foreign language, ask to Wikpedia
#what is the corresponding article of the preferred language, so
#that we can flag it as tagged aswell.
print "\n- Read from the OSM file the articles already tagged"
parseOSMData = ParseOSMData(self)
#list of Wikipedia tags in OSM
self.tagsInOSM = parseOSMData.allTags
self.tagsData = parseOSMData.tagsData
#list of tagged Wikipedia articles
self.taggedTitles = parseOSMData.titles
#tags with errors
self.wrongTags = parseOSMData.wrongTags
#ugly tags (with url, language capitalized...), not errors
self.badTags = parseOSMData.badTags
#add articles manually flagged as tagged in data/workaround/tagged.csv
#in case the parser misses them (strange tags)
self.add_tagged_articles()
if self.args.show_coordinates_from_osm:
print "\n--- Add OSM coordinates to the articles"
parseOSMData.get_centroids()
### Manage Wikipedia data ##############################################
#Read from 'non-mappable' file the categories and articles that
#aren't mappable e.g. "Paintings in the X museum",
#self.nonMappable = {mainCategory.name : {"articles" : [], "subcategories" : []}}
self.nonMappable = self.read_non_mappable_items()
#Check if we have Wikipedia data from Quick Intersection of all the
#categories in the project (config.cfg file)
themesAndCatsNames = wikipedia_downloader.check_catscan_data(self, themesAndCatsNames)
#Organize Wikipedia data.
#self.themes = [Theme(), ...]
# Theme().categories = [Category(), ...]
# Category().subcategories = [Category(), ...]
# Category().articles = [Article(), ...]
#categories without Quick Intersection data
self.categoriesWithoutData = []
allThemes = Themes(self, themesAndCatsNames)
self.themes = allThemes.themesList
#Organize data in regions, for a different visualization
#self.regions = [Region()]
# Region().categories = [Category(), ... ]
self.regions = []
if self.regionsNames != []:
self.regions = Regions(self).regionsList
#Print names of all categories
if self.args.print_categories_list:
self.display_categories_names()
if not self.args.category_info:
#"There's nothing left for me to tell you"
sys.exit(1)
### Merge OSM info into Wikipedia data #################################
#Add to Wikipedia categories and articles istances info about
#their status in OSM: (tagged/not tagged), osm ids and counters
print ("\n- Check which articles are already tagged in the country's "
"OSM file")
for theme in self.themes:
for category in theme.categories:
category.check_articles_in_osm()
self.titlesInOSM, self.titlesNotInOSM = allThemes.lists_of_titles_in_osm_or_not()
#Ask to Wikipedia which articles have/have not Coord template.
#Articles with article.hasTemplate == False will be marked on web pages.
if self.args.show_missing_templates:
print "\n- Check which articles miss geo template (Coord) in Wikipedia"
self.templatesStatus = wikipedia_downloader.read_old_templates_status(self)
wikipedia_downloader.update_templates_status(self)
#Set hasTemplate = False to articles without Coord template
for theme in self.themes:
for category in theme.categories:
category.set_has_template_in_articles()
#If an article is not already tagged in OSM but Wikipedia knows its
#position, it is possible to add a link to zoom to that position
#with JOSM.
if self.args.show_link_to_wikipedia_coordinates:
print "\n- Check the non tagged articles whose position is known by Wikipedia"
wikipedia_downloader.add_wikipedia_coordinates(self)
#Save GeoJSON file with titles and coordinates known by Wikipedia
self.save_titles_with_coords_geojson()
if self.args.infer_coordinates_from_wikipedia:
print "\n- Use Nuts4Nuts to infer coordinates of non tagged articles, whose position is unknown by Wikipedia"
nuts4nuts_infer.infer_coordinates_with_nuts4nuts(self)
#For debugging
# print info about a specific category
if self.args.category_info:
self.print_category_info(self.args.category_info.replace(" ", "_"))
if self.args.create_webpages:
raw_input("\nContinue?[Press any key]")
# write categories trees to text files (uncomment lines)
if self.print_categories_to_text_files == "true":
for theme in self.themes:
for category in theme.categories:
category.print_category_tree_to_file()
#Read and update stats with the number of tagged articles
self.dates, self.days = self.read_past_stats()
download_other_countries = False
self.todayDate, today = self.read_new_stats(download_other_countries)
self.days.append(today)
self.dates.append(self.todayDate)
if len(self.dates) > 1 and self.todayDate == self.dates[-2]:
#This is the second analysis of today.
#Overwrite the previous statistics
del self.dates[-2]
del self.days[-2]
print "\n This is the second time that data ara analyzed today. \
The number of tagged articles will replace that of the lust run in the tags' numbers table."
#Count tags added by each user
self.users = Users(self).users
#Create a json file with the data (needed by non_mappable.html)
tree = {"mappable": True,
"name": "Main",
"size": 1,
"children": []}
for theme in self.themes:
for category in theme.categories:
tree["children"].append(category.build_json_tree())
ifile = open(os.path.join(self.HTMLDIR, "json", "main.json"), "w")
data = json.dumps(tree)
ifile.write(data)
ifile.close()
#Create webpages
if self.args.create_webpages:
# Restrict to the supported locales
self.locales = frozenset(self.SUPPORTED_LOCALES).intersection(
frozenset(self.args.locales))
non_supported_locales = frozenset(self.args.locales) - \
frozenset(self.SUPPORTED_LOCALES)
for locale_langcode in non_supported_locales:
print 'Warning: dropping unsupported locale: {0}'.format(
locale_langcode)
# if no supported locale is chosen fallback to en_US
if not self.locales:
self.locales = frozenset(['en_US'])
for locale_langcode in self.locales:
self.translations = Translations.load("locale",
[locale_langcode]
)
self._ = self.translations.ugettext
print "\n- Create web pages with locale: ", locale_langcode
Creator(self, locale_langcode)
if self.args.browser:
url = os.path.join('html', locale_langcode, 'index.html')
# using .get() suppress stdout output from browser, won't
# suppress stderr
webbrowser.get().open_new(url)
# Create the index.html in the main HTMLDIR to redirect to one
# locales directory
for lang in self.locales:
if self.WIKIPEDIALANG in lang:
Redirect(self, lang)
break
#Save stats
if self.args.save_stats:
self.save_stats_to_csv()
print "\nNew stats have been saved."
else:
print "\nNo stats saved."
#Copy files from html dir to outdir (for example a Dropbox directory)
if self.args.copy:
self.copy_html_files_to_outdir()
print "\nDone."
def __init__(self):
# Options
text = "A partire da una lista di categorie inserite dall'utente nel file 'config', lo script:\
scarica/aggiorna i dati OSM nazionali, scarica da Wikipedia i dati sulle categorie (gli articoli che le compongono)\
e crea delle pagine HTML indicando gli articoli già taggati e da taggare in OSM."
parser = argparse.ArgumentParser(description=text)
group = parser.add_mutually_exclusive_group()
# Manage OSM data
parser.add_argument(
"-d", "--download_osm", help="Scarica i dati OSM nazionali (da Geofabrik)", action="store_true"
)
parser.add_argument(
"-u",
"--update_osm",
help="Aggiorna i dati OSM nazionali scaricati (tramite osmupdate)",
action="store_true",
)
# Analyze data from Wikipedia and OSM
parser.add_argument(
"-a",
"--analyze",
help="Analizza i dati: Wikipedia (sottocategorie ed articoli delle categorie) ed OSM (tag Wikipedia presenti)",
action="store_true",
)
parser.add_argument(
"--category_info",
help="Analizza i dati e stampa le informazioni su una specifica categoria",
action="store",
)
parser.add_argument(
"-t", "--show_missing_templates", help="Segnala gli articoli senza template Coord", action="store_true"
)
parser.add_argument(
"-c",
"--show_link_to_wikipedia_coordinates",
help="Se un articolo non taggato ha delle coordinate su Wikipedia, mostra un link per zoomare sulla sua posizione con JOSM",
action="store_true",
)
parser.add_argument(
"-o",
"--show_coordinates_from_osm",
help="Calcola le coordinate del punto (per i nodi) o del centroide (per way e relations) dell'oggetto",
action="store_true",
)
parser.add_argument(
"-n",
"--infer_coordinates_from_wikipedia",
help="Usa Nuts4Nuts per cercare le coordinate di un articolo non taggato e senza coordinate su Wikipedia",
action="store_true",
)
parser.add_argument(
"-r",
"--update_redirects",
help="Controlla gli articoli delle liste che sono redirects e crea un file con la loro lista per poterli aggiungere manualmente tra i non mappabili, perché non supportati da WIWOSM",
action="store_true",
)
group.add_argument(
"-p",
"--print_categories_list",
help="Analizza i dati e stampa la lista delle categorie nel progetto.",
action="store_true",
)
# Create webpages
group.add_argument(
"-w", "--create_webpages", help="Analizza i dati ed aggiorna le pagine web", action="store_true"
)
parser.add_argument(
"-s",
"--save_stats",
help="Se sono state aggiornate le pagine web, salva il conteggio aggiornato con il numero di articoli taggati (default: chiedi cosa fare).",
action="store_true",
)
parser.add_argument(
"--nofx", help="Non aprire le pagine web in Firefox dopo averle aggiornate.", action="store_true"
)
parser.add_argument(
"--copy",
help="Copia la cartella html nella directory descritta nel file config (es. dir dropbox).",
action="store_true",
)
parser.add_argument("--bitly", help="Use bitly links, to count visits to homepage.", action="store_true")
self.args = parser.parse_args()
if (
self.args.category_info
or self.args.category_info
or self.args.create_webpages
or self.args.print_categories_list
or self.args.update_redirects
or self.args.show_missing_templates
or self.args.show_coordinates_from_osm
):
self.args.analyze = True
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
os.chdir(os.path.dirname(sys.argv[0]))
# Configurations
themesAndCatsNames = self.read_config()
### Manage OpenStreetMap data ##########################################
# Analyse national OSM data file and create lists of already
# tagged Wikipedia articles.
# Download/update OSM data
if self.args.download_osm or self.args.update_osm:
if self.args.download_osm:
OSM.download_osm_data(self)
if self.args.update_osm:
status = OSM.update_osm_data(self)
if self.args.download_osm or (self.args.update_osm and status):
OSM.filter_wikipedia_data_in_osm_file(self)
if self.args.update_osm and not status:
print "I dati OSM erano già aggiornati all'ultimo minuto, o l'aggiornamento con osmupdate è stato interrotto.\
Per ripetere l'aggiornamento, lanciare nuovamente lo script con l'opzione -u."
if not self.args.analyze:
# "There's nothing left for me to tell you"
sys.exit(1)
else:
if not os.path.isfile(self.wOSMFile):
OSM.filter_wikipedia_data_in_osm_file(self)
# Extract Wikipedia articles tagged in OSM with preferred language.
# If an article is tagged in a foreign language, ask to Wikpedia
# what is the corresponding article of the preferred language, so
# that we can flag it as tagged aswell.
print "\n- Estrai dal file OSM gli articoli già taggati"
parseOSMData = ParseOSMData(self)
# list of Wikipedia tags in OSM
self.tagsInOSM = parseOSMData.allTags
self.tagsData = parseOSMData.tagsData
# list of tagged Wikipedia articles
self.taggedTitles = parseOSMData.titles
# tags with errors
self.wrongTags = parseOSMData.wrongTags
# ugly tags (with url, language capitalized...), not errors
self.badTags = parseOSMData.badTags
# add articles manually flagged as tagged in data/workaround/tagged.csv
# in case the parser misses them (strange tags)
self.add_tagged_articles()
if self.args.show_coordinates_from_osm:
print "\n--- Aggiungi le coordinate calcolare da OSM"
parseOSMData.get_centroids()
### Manage Wikipedia data ##############################################
# Read from 'non-mappable' file the categories and articles that
# aren't mappable e.g. "Paintings in the X museum",
# self.nonMappable = {mainCategory.name : {"articles" : [], "subcategories" : []}}
self.nonMappable = self.read_non_mappable_items()
# Check if we have Wikipedia data from CatScan of all the
# categories in the project (config file)
themesAndCatsNames = wikipedia_downloader.check_catscan_data(self, themesAndCatsNames)
# Organize Wikipedia data.
# self.themes = [Theme(), ...]
# Theme().categories = [Category(), ...]
# Category().subcategories = [Category(), ...]
# Category().articles = [Article(), ...]
# categories without catscan data
self.categoriesWithoutData = []
allThemes = Themes(self, themesAndCatsNames)
self.themes = allThemes.themesList
# Organize data in regions, for a different visualization
# self.regions = [Region()]
# Region().categories = [Category(), ... ]
self.regions = Regions(self).regionsList
# Print names of all categories
if self.args.print_categories_list:
self.display_categories_names()
if not self.args.category_info:
# "There's nothing left for me to tell you"
sys.exit(1)
### Merge OSM info into Wikipedia data #################################
# Add to Wikipedia categories and articles istances info about
# their status in OSM: (tagged/not tagged), osm ids and counters
print "\n- Controlla quali articoli nelle liste sono già taggati nel file OSM"
for theme in self.themes:
for category in theme.categories:
category.check_articles_in_osm()
self.titlesInOSM, self.titlesNotInOSM = allThemes.lists_of_titles_in_osm_or_not()
# Ask to Wikipedia which articles have/have not Coord template.
# Articles with article.hasTemplate == False will be marked on web pages.
if self.args.show_missing_templates:
print "\n- Controlla quali articoli non hanno il template Coord in Wikipedia"
self.templatesStatus = wikipedia_downloader.read_old_templates_status(self)
wikipedia_downloader.update_templates_status(self)
# Set hasTemplate = False to articles without Coord template
for theme in self.themes:
for category in theme.categories:
category.set_has_template_in_articles()
# If an article is not already tagged in OSM but Wikipedia knows its
# position, it is possible to add a link to zoom to that position
# with JOSM.
if self.args.show_link_to_wikipedia_coordinates:
print "\n- Controlla di quali articoli non taggati Wikipedia conosce già la posizione"
wikipedia_downloader.add_wikipedia_coordinates(self)
# Save GeoJSON file with titles and coordinates known by Wikipedia
self.save_titles_with_coords_geojson()
if self.args.infer_coordinates_from_wikipedia:
print "\n- Usa Nuts4Nuts per inferire la posizione di alcuni articoli"
nuts4nuts_infer.infer_coordinates_with_nuts4nuts(self)
# Download from Wikipedia the lists of redirects and create a file
# so that they can then be manually copied to non_mappable file
if self.args.update_redirects:
wikipedia_downloader.find_redirects(self)
# For debugging
# print info about a specific category
if self.args.category_info:
self.print_category_info(self.args.category_info.replace(" ", "_"))
if self.args.create_webpages:
raw_input("\nContinue?[Press any key]")
# write categories trees to text files (uncomment lines)
if self.print_categories_to_text_files == "true":
for theme in self.themes:
for category in theme.categories:
category.print_category_tree_to_file()
# Read and update stats with the number of tagged articles
self.dates, self.days = self.read_past_stats()
download_other_countries = False
self.todayDate, today = self.read_new_stats(download_other_countries)
self.days.append(today)
self.dates.append(self.todayDate)
if len(self.dates) > 1 and self.todayDate == self.dates[-2]:
# This is the second analysis of today.
# Overwrite the previous statistics
del self.dates[-2]
del self.days[-2]
print "\n Questa è la seconda volta che i dati vengono analizzati oggi. \
Il numero di articoli taggati sostituisce quelli precedenti nella tabella dei conteggi."
# Count tags added by each user
self.users = Users(self).users
# Create a json file with the data (needed by non_mappable.html)
tree = {"mappable": True, "name": "Main", "size": 1, "children": []}
for theme in self.themes:
for category in theme.categories:
tree["children"].append(category.build_json_tree())
ifile = open(os.path.join(self.HTMLDIR, "json", "main.json"), "w")
data = json.dumps(tree)
ifile.write(data)
ifile.close()
# Create webpages
if self.args.create_webpages:
print "\n- Crea pagine web"
Creator(self)
# Save stats
if self.args.create_webpages and self.args.save_stats:
answer = "y"
else:
answer = raw_input(
"\n- Salvo il numero di articoli mappati/da mappare in './data/stats/stats.csv'?\n [y/N]\n"
)
if answer in ("y", "Y"):
self.save_stats_to_csv()
else:
print "\nI nuovi conteggi non vengono salvati."
# Copy files from html dir to outdir (for example a Dropbox directory)
if self.args.copy:
self.copy_html_files_to_outdir()
print "\nDone."
def __init__(self):
#Options
text = "A partire da una lista di categorie inserite dall'utente nel file 'config', lo script:\
scarica/aggiorna i dati OSM nazionali, scarica da Wikipedia i dati sulle categorie (gli articoli che le compongono)\
e crea delle pagine HTML indicando gli articoli già taggati e da taggare in OSM."
parser = argparse.ArgumentParser(description=text)
group = parser.add_mutually_exclusive_group()
#Manage OSM data
parser.add_argument("-d",
"--download_osm",
help="Scarica i dati OSM nazionali (da Geofabrik)",
action="store_true")
parser.add_argument(
"-u",
"--update_osm",
help="Aggiorna i dati OSM nazionali scaricati (tramite osmupdate)",
action="store_true")
#Analyze data from Wikipedia and OSM
parser.add_argument(
"-a",
"--analyze",
help=
"Analizza i dati: Wikipedia (sottocategorie ed articoli delle categorie) ed OSM (tag Wikipedia presenti)",
action="store_true")
parser.add_argument(
"--category_info",
help=
"Analizza i dati e stampa le informazioni su una specifica categoria",
action="store")
parser.add_argument("-t",
"--show_missing_templates",
help="Segnala gli articoli senza template Coord",
action="store_true")
parser.add_argument(
"-c",
"--show_link_to_wikipedia_coordinates",
help=
"Se un articolo non taggato ha delle coordinate su Wikipedia, mostra un link per zoomare sulla sua posizione con JOSM",
action="store_true")
parser.add_argument(
"-o",
"--show_coordinates_from_osm",
help=
"Calcola le coordinate del punto (per i nodi) o del centroide (per way e relations) dell'oggetto",
action="store_true")
parser.add_argument(
"-n",
"--infer_coordinates_from_wikipedia",
help=
"Usa Nuts4Nuts per cercare le coordinate di un articolo non taggato e senza coordinate su Wikipedia",
action="store_true")
parser.add_argument(
"-r",
"--update_redirects",
help=
"Controlla gli articoli delle liste che sono redirects e crea un file con la loro lista per poterli aggiungere manualmente tra i non mappabili, perché non supportati da WIWOSM",
action="store_true")
group.add_argument(
"-p",
"--print_categories_list",
help=
"Analizza i dati e stampa la lista delle categorie nel progetto.",
action="store_true")
#Create webpages
group.add_argument("-w",
"--create_webpages",
help="Analizza i dati ed aggiorna le pagine web",
action="store_true")
parser.add_argument(
"-s",
"--save_stats",
help=
"Se sono state aggiornate le pagine web, salva il conteggio aggiornato con il numero di articoli taggati (default: chiedi cosa fare).",
action="store_true")
parser.add_argument(
"--nofx",
help="Non aprire le pagine web in Firefox dopo averle aggiornate.",
action="store_true")
parser.add_argument(
"--copy",
help=
"Copia la cartella html nella directory descritta nel file config (es. dir dropbox).",
action="store_true")
parser.add_argument(
"--bitly",
help="Use bitly links, to count visits to homepage.",
action="store_true")
self.args = parser.parse_args()
if self.args.category_info or self.args.category_info\
or self.args.create_webpages or self.args.print_categories_list\
or self.args.update_redirects\
or self.args.show_missing_templates\
or self.args.show_coordinates_from_osm:
self.args.analyze = True
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
os.chdir(os.path.dirname(sys.argv[0]))
#Configurations
themesAndCatsNames = self.read_config()
### Manage OpenStreetMap data ##########################################
#Analyse national OSM data file and create lists of already
#tagged Wikipedia articles.
#Download/update OSM data
if self.args.download_osm or self.args.update_osm:
if self.args.download_osm:
OSM.download_osm_data(self)
if self.args.update_osm:
status = OSM.update_osm_data(self)
if self.args.download_osm or (self.args.update_osm and status):
OSM.filter_wikipedia_data_in_osm_file(self)
if self.args.update_osm and not status:
print "I dati OSM erano già aggiornati all'ultimo minuto, o l'aggiornamento con osmupdate è stato interrotto.\
Per ripetere l'aggiornamento, lanciare nuovamente lo script con l'opzione -u."
if not self.args.analyze:
#"There's nothing left for me to tell you"
sys.exit(1)
else:
if not os.path.isfile(self.wOSMFile):
OSM.filter_wikipedia_data_in_osm_file(self)
#Extract Wikipedia articles tagged in OSM with preferred language.
#If an article is tagged in a foreign language, ask to Wikpedia
#what is the corresponding article of the preferred language, so
#that we can flag it as tagged aswell.
print "\n- Estrai dal file OSM gli articoli già taggati"
parseOSMData = ParseOSMData(self)
#list of Wikipedia tags in OSM
self.tagsInOSM = parseOSMData.allTags
self.tagsData = parseOSMData.tagsData
#list of tagged Wikipedia articles
self.taggedTitles = parseOSMData.titles
#tags with errors
self.wrongTags = parseOSMData.wrongTags
#ugly tags (with url, language capitalized...), not errors
self.badTags = parseOSMData.badTags
#add articles manually flagged as tagged in data/workaround/tagged.csv
#in case the parser misses them (strange tags)
self.add_tagged_articles()
if self.args.show_coordinates_from_osm:
print "\n--- Aggiungi le coordinate calcolare da OSM"
parseOSMData.get_centroids()
### Manage Wikipedia data ##############################################
#Read from 'non-mappable' file the categories and articles that
#aren't mappable e.g. "Paintings in the X museum",
#self.nonMappable = {mainCategory.name : {"articles" : [], "subcategories" : []}}
self.nonMappable = self.read_non_mappable_items()
#Check if we have Wikipedia data from CatScan of all the
#categories in the project (config file)
themesAndCatsNames = wikipedia_downloader.check_catscan_data(
self, themesAndCatsNames)
#Organize Wikipedia data.
#self.themes = [Theme(), ...]
# Theme().categories = [Category(), ...]
# Category().subcategories = [Category(), ...]
# Category().articles = [Article(), ...]
#categories without catscan data
self.categoriesWithoutData = []
allThemes = Themes(self, themesAndCatsNames)
self.themes = allThemes.themesList
#Organize data in regions, for a different visualization
#self.regions = [Region()]
# Region().categories = [Category(), ... ]
self.regions = Regions(self).regionsList
#Print names of all categories
if self.args.print_categories_list:
self.display_categories_names()
if not self.args.category_info:
#"There's nothing left for me to tell you"
sys.exit(1)
### Merge OSM info into Wikipedia data #################################
#Add to Wikipedia categories and articles istances info about
#their status in OSM: (tagged/not tagged), osm ids and counters
print "\n- Controlla quali articoli nelle liste sono già taggati nel file OSM"
for theme in self.themes:
for category in theme.categories:
category.check_articles_in_osm()
self.titlesInOSM, self.titlesNotInOSM = allThemes.lists_of_titles_in_osm_or_not(
)
#Ask to Wikipedia which articles have/have not Coord template.
#Articles with article.hasTemplate == False will be marked on web pages.
if self.args.show_missing_templates:
print "\n- Controlla quali articoli non hanno il template Coord in Wikipedia"
self.templatesStatus = wikipedia_downloader.read_old_templates_status(
self)
wikipedia_downloader.update_templates_status(self)
#Set hasTemplate = False to articles without Coord template
for theme in self.themes:
for category in theme.categories:
category.set_has_template_in_articles()
#If an article is not already tagged in OSM but Wikipedia knows its
#position, it is possible to add a link to zoom to that position
#with JOSM.
if self.args.show_link_to_wikipedia_coordinates:
print "\n- Controlla di quali articoli non taggati Wikipedia conosce già la posizione"
wikipedia_downloader.add_wikipedia_coordinates(self)
#Save GeoJSON file with titles and coordinates known by Wikipedia
self.save_titles_with_coords_geojson()
if self.args.infer_coordinates_from_wikipedia:
print "\n- Usa Nuts4Nuts per inferire la posizione di alcuni articoli"
nuts4nuts_infer.infer_coordinates_with_nuts4nuts(self)
#Download from Wikipedia the lists of redirects and create a file
#so that they can then be manually copied to non_mappable file
if self.args.update_redirects:
wikipedia_downloader.find_redirects(self)
#For debugging
# print info about a specific category
if self.args.category_info:
self.print_category_info(self.args.category_info.replace(" ", "_"))
if self.args.create_webpages:
raw_input("\nContinue?[Press any key]")
# write categories trees to text files (uncomment lines)
if self.print_categories_to_text_files == "true":
for theme in self.themes:
for category in theme.categories:
category.print_category_tree_to_file()
#Read and update stats with the number of tagged articles
self.dates, self.days = self.read_past_stats()
download_other_countries = False
self.todayDate, today = self.read_new_stats(download_other_countries)
self.days.append(today)
self.dates.append(self.todayDate)
if len(self.dates) > 1 and self.todayDate == self.dates[-2]:
#This is the second analysis of today.
#Overwrite the previous statistics
del self.dates[-2]
del self.days[-2]
print "\n Questa è la seconda volta che i dati vengono analizzati oggi. \
Il numero di articoli taggati sostituisce quelli precedenti nella tabella dei conteggi."
#Count tags added by each user
self.users = Users(self).users
#Create a json file with the data (needed by non_mappable.html)
tree = {"mappable": True, "name": "Main", "size": 1, "children": []}
for theme in self.themes:
for category in theme.categories:
tree["children"].append(category.build_json_tree())
ifile = open(os.path.join(self.HTMLDIR, "json", "main.json"), "w")
data = json.dumps(tree)
ifile.write(data)
ifile.close()
#Create webpages
if self.args.create_webpages:
print "\n- Crea pagine web"
Creator(self)
#Save stats
if self.args.create_webpages and self.args.save_stats:
answer = "y"
else:
answer = raw_input(
"\n- Salvo il numero di articoli mappati/da mappare in './data/stats/stats.csv'?\n [y/N]\n"
)
if answer in ("y", "Y"):
self.save_stats_to_csv()
else:
print "\nI nuovi conteggi non vengono salvati."
#Copy files from html dir to outdir (for example a Dropbox directory)
if self.args.copy:
self.copy_html_files_to_outdir()
print "\nDone."
