def make_news_bin(mode, console):
global system, dictionaries, topics_news, languages, make_news, data, country_code, language_code, locations_data, header, topics_table, articles_table, source_table, locations_table, pictures_table
if mode == "ap_english":
topics_news = collections.OrderedDict()
topics_news["National News"] = "national"
topics_news["International News"] = "world"
topics_news["Sports"] = "sports"
topics_news["Arts/Entertainment"] = "entertainment"
topics_news["Business"] = "business"
topics_news["Science/Health"] = "science"
topics_news["Technology"] = "technology"
topics_news["Oddities"] = "oddities"
languages = [1, 3, 4]
language_code = 1
country_code = 49
elif mode == "ap_spanish":
topics_news = collections.OrderedDict()
topics_news["Generales"] = "general"
topics_news["Financieras"] = "finance"
topics_news["Deportivas"] = "sports"
topics_news["Espectáculos"] = "shows"
languages = [1, 3, 4]
language_code = 4
country_code = 49
elif mode == "reuters_europe_english":
topics_news = collections.OrderedDict()
topics_news["World"] = "world"
topics_news["UK"] = "uk"
topics_news["Health"] = "health"
topics_news["Science"] = "science"
topics_news["Technology"] = "technology"
topics_news["Entertainment"] = "entertainment"
topics_news["Sports"] = "sports"
topics_news["Lifestyle"] = "lifestyle"
languages = [1, 2, 3, 4, 5, 6]
language_code = 1
country_code = 110
elif mode == "afp_french":
topics_news = collections.OrderedDict()
topics_news["Monde"] = "world"
topics_news["Sport"] = "sports"
topics_news["Societe"] = "society"
topics_news["Culture"] = "culture"
topics_news["Economie"] = "economy"
topics_news["Politique"] = "politics"
languages = [1, 2, 3, 4, 5, 6]
language_code = 3
country_code = 110
elif mode == "donaukurier_german":
topics_news = collections.OrderedDict()
topics_news["Nachrichten"] = "world"
topics_news["Wirtschaft"] = "economy"
topics_news["Kultur"] = "culture"
languages = [1, 2, 3, 4, 5, 6]
language_code = 2
country_code = 110
elif mode == "ansa_italian":
topics_news = collections.OrderedDict()
topics_news["Dal mondo"] = "world"
topics_news["Sport"] = "sports"
topics_news["Economia"] = "economy"
topics_news["Tecnologia"] = "technology"
topics_news["Cultura"] = "culture"
languages = [1, 2, 3, 4, 5, 6]
language_code = 5
country_code = 110
elif mode == "nu_dutch":
topics_news = collections.OrderedDict()
topics_news["Algemeen"] = "general"
topics_news["Economie"] = "economy"
topics_news["Sport"] = "sports"
topics_news["Tech"] = "technology"
topics_news["Entertainment"] = "entertainment"
topics_news["Lifestyle"] = "lifestyle"
topics_news["Opmerkelijk"] = "noteworthy"
languages = [1, 2, 3, 4, 5, 6]
language_code = 6
country_code = 110
elif mode == "reuters_japanese":
topics_news = collections.OrderedDict()
topics_news["ワールド"] = "world"
topics_news["ビジネス"] = "business"
topics_news["スポーツ"] = "sports"
topics_news["テクノロジー"] = "technology"
topics_news["エンタテインメント"] = "entertainment"
languages = [0]
language_code = 0
country_code = 1
numbers = 0
if not os.path.exists("newstime"):
os.mkdir("newstime")
for topics in topics_news.values():
newstime = collections.OrderedDict()
for keys in data.keys():
if topics in keys:
numbers += 1
newstime[data[keys][3]] = get_timestamp(1) + u32(numbers)
pickle.dump(
newstime,
open(
"newstime/newstime.%s-%s-%s-%s" %
(str(datetime.now().hour).zfill(2), mode, topics, console),
"wb"))
dictionaries = []
header = collections.OrderedDict()
topics_table = collections.OrderedDict()
articles_table = collections.OrderedDict()
source_table = collections.OrderedDict()
locations_table = collections.OrderedDict()
pictures_table = collections.OrderedDict()
locations_data = newsdownload.locations_download(language_code, data)
make_header(0, 30)
make_wiimenu_articles()
make_topics_table()
make_timestamps_table(mode)
make_articles_table(mode)
make_source_table()
make_locations_table()
make_pictures_table()
make_articles()
make_topics(0, 0)
make_source_name_copyright()
make_locations()
make_source_pictures()
make_pictures()
make_riiconnect24_text()
write_dictionary(mode)
headlines = []
for article in data.values():
if article[3].decode("utf-16be") not in headlines:
headlines.append(article[3].decode("utf-16be") + "\n")
make_news = "".join(headlines)
return make_news
def make_news_bin(mode, console, data):
if mode == "ap_english":
topics_news = collections.OrderedDict()
topics_news["National News"] = "national"
topics_news["International News"] = "world"
topics_news["Sports"] = "sports"
topics_news["Arts/Entertainment"] = "entertainment"
topics_news["Business"] = "business"
topics_news["Science/Health"] = "science"
topics_news["Technology"] = "technology"
languages = [1, 3, 4]
language_code = 1
country_code = 49
elif mode == "ap_spanish":
topics_news = collections.OrderedDict()
topics_news["Generales"] = "general"
topics_news["Financieras"] = "finance"
topics_news["Deportivas"] = "sports"
topics_news["Espectáculos"] = "shows"
languages = [1, 3, 4]
language_code = 4
country_code = 49
elif mode == "reuters_english":
topics_news = collections.OrderedDict()
topics_news["World"] = "world"
topics_news["UK"] = "uk"
topics_news["Health"] = "health"
topics_news["Science"] = "science"
topics_news["Technology"] = "technology"
topics_news["Oddly Enough"] = "offbeat"
topics_news["Entertainment"] = "entertainment"
topics_news["Sports"] = "sports"
languages = [1, 2, 3, 4, 5, 6]
language_code = 1
country_code = 110
elif mode == "zeit_german":
topics_news = collections.OrderedDict()
topics_news["General"] = "general"
topics_news["Politik"] = "politics"
topics_news["Wirtschaft"] = "economy"
topics_news["Gesellschaft"] = "society"
topics_news["Kultur"] = "culture"
topics_news["Wissen"] = "knowledge"
topics_news["Digital"] = "digital"
topics_news["Sport"] = "sports"
languages = [1, 2, 3, 4, 5, 6]
language_code = 2
country_code = 110
elif mode == "afp_french":
topics_news = collections.OrderedDict()
topics_news["Top News"] = "topnews"
topics_news["Société"] = "society"
topics_news["Monde"] = "world"
topics_news["Politique"] = "politique"
languages = [1, 2, 3, 4, 5, 6]
language_code = 3
country_code = 110
elif mode == "ansa_italian":
topics_news = collections.OrderedDict()
topics_news["Dal mondo"] = "world"
topics_news["Dall'Italia"] = "italy"
topics_news["Sport"] = "sports"
topics_news["Economia"] = "economy"
topics_news["Cultura"] = "culture"
languages = [1, 2, 3, 4, 5, 6]
language_code = 5
country_code = 110
elif mode == "anp_dutch":
topics_news = collections.OrderedDict()
topics_news["Algemeen"] = "general"
topics_news["Economie"] = "economy"
topics_news["Sport"] = "sports"
topics_news["Tech"] = "technology"
topics_news["Entertainment"] = "entertainment"
topics_news["Lifestyle"] = "lifestyle"
topics_news["Opmerkelijk"] = "noteworthy"
languages = [1, 2, 3, 4, 5, 6]
language_code = 6
country_code = 110
elif mode == "news24_mainichi_japanese":
topics_news = collections.OrderedDict()
topics_news["政治"] = "politics"
topics_news["経済"] = "economy"
topics_news["国際"] = "international"
topics_news["社会"] = "society"
topics_news["スポーツ"] = "sports"
topics_news["芸能文化"] = "entertainment"
languages = [0]
language_code = 0
country_code = 1
global system, dictionaries
system = console
numbers = 0
if not os.path.exists("newstime"): os.mkdir("newstime")
for topics in topics_news.values():
newstime = collections.OrderedDict()
for keys in data.keys():
if topics in keys:
numbers += 1
newstime[data[keys][3]] = get_timestamp(1) + u32(numbers)
pickle.dump(
newstime,
open(
"newstime/" + "newstime." + str(datetime.now().hour).zfill(2) +
"-" + mode + "-" + topics + "-" + system, "w+"))
dictionaries = []
locations_data = newsdownload.locations_download(language_code, data)
header = make_header(country_code, languages, language_code, 0, 30, data)
wiimenu_articles = make_wiimenu_articles(header, data)
topics_table = make_topics_table(header, topics_news, data)
timestamps_table = make_timestamps_table(topics_table, topics_news, mode,
data)
articles_table = make_articles_table(header, locations_data, data)
source_table = make_source_table(header, articles_table, data)
locations_table = make_locations_table(header, locations_data)
pictures_table = make_pictures_table(header, data)
articles = make_articles(pictures_table, articles_table, data)
topics = make_topics(topics_table, source_table, 0, topics_news, 0)
source_name_copyright = make_source_name_copyright(source_table,
language_code, data)
locations = make_locations(locations_table, locations_data)
source_pictures = make_source_pictures(source_table, data)
pictures = make_pictures(pictures_table, data)
riiconnect24_text = make_riiconnect24_text()
write = write_dictionary(mode)
headlines = []
for article in data.values():
if article[3].decode("utf-16be") not in headlines:
headlines.append(article[3].decode("utf-16be") + "\n")
return "".join(headlines)
def process_news(name, mode, language, countries, d):
print "News Channel File Generator \nBy Larsen Vallecillo / www.rc24.xyz\n\nMaking news.bin for %s...\n" % name
global language_code, system
language_code = language
data = d.newsdata
"""If there are more than 25 news articles, delete the rest. This is so the file doesn't get too large."""
i = 0
for key in data.keys():
i += 1
if i > 25:
del data[key]
# data = remove_duplicates(data)
data = remove_duplicates2(data)
locations_data = newsdownload.locations_download(language_code, data)
for system in ["wii", "wii_u"]:
make_news = make_news_bin(mode, system, data, locations_data)
if config["production"]:
"""Log stuff to Datadog."""
statsd.increment("news.total_files_built")
"""This will use a webhook to log that the script has been ran."""
webhook = {
"username":
"******",
"content":
"News Data has been updated!",
"avatar_url":
"https://rc24.xyz/images/logo-small.png",
"attachments": [{
"fallback":
"News Data Update",
"color":
"#1B691E",
"author_name":
"RiiConnect24 News Script",
"author_icon":
"https://rc24.xyz/images/webhooks/news/profile.png",
"text":
make_news,
"title":
"Update!",
"fields": [{
"title": "Script",
"value": "News Channel (" + name + ")",
"short": "false"
}],
"thumb_url":
"https://rc24.xyz/images/webhooks/news/%s.png" % mode,
"footer":
"RiiConnect24 Script",
"footer_icon":
"https://rc24.xyz/images/logo-small.png",
"ts":
int(time.mktime(datetime.utcnow().timetuple())) + 25200
}]
}
for url in config["webhook_urls"]:
requests.post(url, json=webhook, allow_redirects=True)
filesize = sum(
os.path.getsize(f) - 320 for f in
glob.glob("/var/www/wapp.wii.com/news/v2/%s/%s/news.bin.*" %
(language_code, countries[0])))
if filesize > 3712000:
log("News files exceed the maximum file size amount.", "error")
for country in countries:
copy_file(mode, "wii", country)
copy_file(mode, "wii_u", country)
newsfilename = "news.bin." + str(
datetime.utcnow().hour).zfill(2) + "." + mode + "."
os.remove(newsfilename + "wii")
os.remove(newsfilename + "wii_u")