def make_worldwide_result_table(header):
table = collections.OrderedDict()
dictionaries.append(table)
worldwide_detailed_table_count_all = 0
header["worldwide_result_offset"] = offset_count()
for i in results:
if results[i][8] == "w":
worldwide_detailed_table_count = 0
for j in range(len(countries)): # 33
total = 0
for voters in range(0, 4):
total += results[i][voters][j]
if total > 0:
worldwide_detailed_table_count += 1
table["poll_id_%s" % num()] = u32(i)
table["male_voters_response_1_num_%s" % num()] = u32(
sum(results[i][0]))
table["male_voters_response_2_num_%s" % num()] = u32(
sum(results[i][2]))
table["female_voters_response_1_num_%s" % num()] = u32(
sum(results[i][1]))
table["female_voters_response_2_num_%s" % num()] = u32(
sum(results[i][3]))
table["predictors_response_1_num_%s" % num()] = u32(
sum(results[i][4]))
table["predictors_response_2_num_%s" % num()] = u32(
sum(results[i][5]))
table["total_worldwide_detailed_tables_%s" %
num()] = u8(worldwide_detailed_table_count)
table["starting_worldwide_detailed_table_number_%s" %
num()] = u32(worldwide_detailed_table_count_all)
worldwide_detailed_table_count_all += worldwide_detailed_table_count
return table
def make_topics_table(header, topics_news):
topics_table = collections.OrderedDict()
dictionaries.append(topics_table)
header["topics_offset"] = offset_count() # Offset for the topics table.
topics_table["new_topics_offset"] = u32(0) # Offset for the newest topic.
topics_table["new_topics_article_size"] = u32(0) # Size for the amount of articles to choose for the newest topic.
topics_table["new_topics_article_offset"] = u32(0) # Offset for the articles to choose for the newest topic.
numbers = 0
for _ in list(topics_news.values()):
numbers += 1
topics_table["topics_%s_offset" % str(numbers)] = u32(0) # Offset for the topic.
topics_table["topics_%s_article_number" % str(numbers)] = u32(0) # Number of articles that will be in a certain topic.
topics_table["topics_%s_article_offset" % str(numbers)] = u32(0) # Offset for the articles to choose for the topic.
header["topics_number"] = u32(numbers + 1) # Number of entries for the topics table.
return topics_table
def make_source_pictures(source_table, data):
source_pictures = collections.OrderedDict()
dictionaries.append(source_pictures)
source_articles = []
sources = ["ANP", "AP", "dpa", "Reuters", "SID", "NU.nl"] # these are the news sources which will use a custom JPG for the logo
for article in list(data.values()):
if article[8] not in source_articles:
if article[8] in sources:
source_articles.append(article[8])
source_table["pictures_offset_%s" % article[8]] = offset_count()
with open("./Channels/News_Channel/logos/%s.jpg" % article[8], "rb") as source_file:
image = source_pictures["logo_%s" % article[8]] = source_file.read()
source_table["pictures_size_%s" % article[8]] = u32(len(image))
if source_table["source_picture_%s" % article[8]] != u8(0):
source_table["source_picture_%s" % article[8]] = u8(0)
return source_pictures
def make_pictures_table(header, data):
pictures_table = {}
dictionaries.append(pictures_table)
header["pictures_offset"] = offset_count() # Offset for the pictures table.
pictures_number = 0
numbers = 0
for article in list(data.values()):
numbers += 1
if article[4] is not None:
if article[5] is not None:
pictures_table["credits_%s_size" % numbers] = u32(
len(article[5])
) # Size of the credits.
pictures_table["credits_%s_offset" % numbers] = u32(
0
) # Offset for the credits.
else:
pictures_table["credits_%s_size" % numbers] = u32(
0
) # Size of the credits.
pictures_table["credits_%s_offset" % numbers] = u32(
0
) # Offset for the credits.
if article[6] is not None:
pictures_table["captions_%s_size" % numbers] = u32(
len(article[6])
) # Size of the captions.
pictures_table["captions_%s_offset" % numbers] = u32(
0
) # Offset for the captions.
else:
pictures_table["captions_%s_size" % numbers] = u32(
0
) # Size of the credits.
pictures_table["captions_%s_offset" % numbers] = u32(
0
) # Offset for the captions.
pictures_number += 1
pictures_table["pictures_%s_size" % numbers] = u32(
len(article[4])
) # Size of the pictures.
pictures_table["pictures_%s_offset" % numbers] = u32(
0
) # Offset for the pictures.
header["pictures_number"] = u32(
pictures_number
) # Number of entries for the pictures table.
if config["production"]:
statsd.increment("news.total_pictures", pictures_number)
return pictures_table
def make_articles_table(mode, locations_data, header, data):
articles_table = {}
dictionaries.append(articles_table)
p_number = 0
numbers = 0
header["articles_offset"] = offset_count()
for keys, article in list(data.items()):
numbers += 1
articles_table["article_%s_number" % numbers] = u32(
numbers
) # Number for the article.
articles_table["source_%s_number" % numbers] = u32(0) # Number for the source.
articles_table["location_%s_number" % numbers] = u32(
4294967295
) # Number for the location.
for locations in list(locations_data.keys()):
for article_name in locations_data[locations][2]:
if keys == article_name:
articles_table["location_%s_number" % numbers] = u32(
list(locations_data.keys()).index(locations)
) # Number for the location.
if article[4] is not None:
articles_table["term_timestamp_%s" % numbers] = get_timestamp(
1
) # Timestamp for the term.
articles_table["picture_%s_number" % numbers] = u32(
p_number
) # Number for the picture.
p_number += 1
else:
articles_table["term_timestamp_%s" % numbers] = u32(
0
) # Timestamp for the term.
articles_table["picture_%s_number" % numbers] = u32(
4294967295
) # Number for the picture.
articles_table["published_time_%s" % numbers] = article[0] # Published time.
articles_table["updated_time_%s" % numbers] = get_timestamp(1) # Updated time.
articles_table["headline_%s_size" % numbers] = u32(
len(article[3].replace(b"\n", b""))
) # Size of the headline.
articles_table["headline_%s_offset" % numbers] = u32(
0
) # Offset for the headline.
articles_table["article_%s_size" % numbers] = u32(
len(article[2])
) # Size of the article.
articles_table["article_%s_offset" % numbers] = u32(
0
) # Offset for the article.
header["articles_number"] = u32(
numbers
) # Number of entries for the articles table.
if config["production"]:
statsd.increment("news.total_articles", numbers)
statsd.increment("news.articles." + mode, numbers)
return articles_table
def make_header(data):
header = {}
dictionaries.append(header)
header["updated_timestamp_1"] = get_timestamp(1) # Updated time.
header["term_timestamp"] = get_timestamp(2) # Timestamp for the term.
header["country_code"] = u32_littleendian(country_code) # Wii Country Code.
header["updated_timestamp_2"] = get_timestamp(1) # 3rd timestamp.
# List of languages that appear on the language select screen
numbers = 0
for language in languages:
numbers += 1
header["language_select_%s" % numbers] = u8(language)
# Fills the rest of the languages as null
while numbers < 16:
numbers += 1
header["language_select_%s" % numbers] = u8(255)
header["language_code"] = u8(language_code) # Wii language code.
header["goo_flag"] = u8(0) # Flag to make the Globe display "Powered by Goo".
header["language_select_screen_flag"] = u8(
0
) # Flag to bring up the language select screen.
header["download_interval"] = u8(
30
) # Interval in minutes to check for new articles to display on the Wii Menu.
header["message_offset"] = u32(0) # Offset for a message.
header["topics_number"] = u32(0) # Number of entries for the topics table.
header["topics_offset"] = u32(0) # Offset for the topics table.
header["articles_number"] = u32(0) # Number of entries for the articles table.
header["articles_offset"] = u32(0) # Offset for the articles table.
header["source_number"] = u32(0) # Number of entries for the source table.
header["source_offset"] = u32(0) # Offset for the source table.
header["locations_number"] = u32(0) # Number of entries for the locations.
header["locations_offset"] = u32(0) # Offset for the locations table.
header["pictures_number"] = u32(0) # Number of entries for the pictures table.
header["pictures_offset"] = u32(0) # Offset for the pictures table.
header["count"] = u16(480) # Count value.
header["unknown"] = u16(0) # Unknown.
header["wiimenu_articles_number"] = u32(0) # Number of Wii Menu article entries.
header["wiimenu_articles_offset"] = u32(0) # Offset for the Wii Menu article table.
header[
"wiimenu_articles_offset"
] = offset_count() # Offset for the Wii Menu article table.
numbers = 0
headlines = []
for article in list(data.values()):
if numbers < 11:
if article[3].replace(b"\n", b"") not in headlines:
numbers += 1
headlines.append(article[3])
header["headline_%s_size" % numbers] = u32(0) # Size of the headline.
header["headline_%s_offset" % numbers] = u32(
0
) # Offset for the headline.
return header
def make_news_bin(mode, data, locations_data):
global dictionaries, languages, country_code, language_code
source = sources[mode]
if source is None:
print("Could not find %s in sources.")
topics_news = source["topics_news"]
languages = source["languages"]
language_code = source["language_code"]
country_code = source["country_code"]
numbers = 0
if not os.path.exists("newstime"):
os.mkdir("newstime")
for topics in list(topics_news.values()):
newstime = {}
for keys in list(data.keys()):
if topics in keys:
numbers += 1
newstime[data[keys][3]] = get_timestamp(1) + u32(numbers)
pickle.dump(
newstime,
open(
"newstime/newstime.%s-%s-%s-wii"
% (str(datetime.now().hour).zfill(2), mode, topics),
"wb",
),
)
dictionaries = []
# ton of functions to make news
header = make_header(data)
make_wiimenu_articles(header, data)
topics_table = make_topics_table(header, topics_news)
make_timestamps_table(mode, topics_table, topics_news)
articles_table = make_articles_table(mode, locations_data, header, data)
source_table = make_source_table(header, articles_table, source, data)
locations_table = make_locations_table(header, locations_data)
pictures_table = make_pictures_table(header, data)
make_articles(articles_table, pictures_table, data)
make_topics(topics_table, topics_news)
make_source_name_copyright(source_table, source, data)
make_locations(locations_data, locations_table)
make_source_pictures(source_table, data)
make_pictures(pictures_table, data)
make_riiconnect24_text()
write_dictionary(mode)
headlines = []
for article in list(data.values()):
try:
if article[3].replace(b"\n", b"").decode("utf-16be") not in headlines:
headlines.append(
article[3].replace(b"\n", b"").decode("utf-16be") + "\n"
)
except UnicodeDecodeError:
pass
make_news = "".join(headlines)
return make_news
def offset_count():
return u32(12 + sum(
len(values) for dictionary in dictionaries
for values in dictionary.values() if values))
def get_timestamp(mode):
if mode == 1:
return u32(int((calendar.timegm(datetime.utcnow().timetuple()) - 946684800) / 60))
elif mode == 2:
return u32(int((calendar.timegm(datetime.utcnow().timetuple()) - 946684800) / 60) + 1500)
def make_question_text_table(header):
global questions
question_text_table = collections.OrderedDict()
dictionaries.append(question_text_table)
header["question_offset"] = offset_count()
for q in question_keys:
if not is_worldwide(q):
list = country_language[country_code]
elif is_worldwide(q):
if file_type == "v":
list = country_language[country_code]
elif file_type == "q":
list = range(1, 9)
for language_code in list:
if get_question(q, language_code) is not None:
num = question_keys.index(q)
question_text_table["language_code_%s_%s" % (num, language_code)] = u8(language_code)
question_text_table["question_offset_%s_%s" % (num, language_code)] = u32(0)
question_text_table["response_1_offset_%s_%s" % (num, language_code)] = u32(0)
question_text_table["response_2_offset_%s_%s" % (num, language_code)] = u32(0)
return question_text_table
def make_source_name_copyright(source_table, data):
source_name_copyright = collections.OrderedDict()
dictionaries.append(source_name_copyright)
sources = []
source_names = {}
"""Text for the copyright. Some of these I had to make up, because if you don't specify a copyright there will be a line that will be in the way in the news article."""
copyrights = {
"AP":
("Copyright %s The Associated Press. All rights reserved. This material may not be published, broadcast, rewritten or redistributed."
% date.today().year).decode("utf-8").encode("utf-16be"),
"Reuters":
("© %s Thomson Reuters. All rights reserved. Republication or redistribution of Thomson Reuters content, including by framing or similar means, is prohibited without the prior written consent of Thomson Reuters. Thomson Reuters and the Kinesis logo are trademarks of Thomson Reuters and its affiliated companies."
% date.today().year).decode("utf-8").encode("utf-16be"),
"AFP":
("All reproduction and representation rights reserved. © %s Agence France-Presse"
% date.today().year).decode("utf-8").encode("utf-16be"),
"AFP_French":
("Tous droits de reproduction et de diffusion réservés. © %s Agence France-Presse"
% date.today().year).decode("utf-8").encode("utf-16be"),
"ANP":
("All reproduction and representation rights reserved. © %s B.V. Algemeen Nederlands Persbureau ANP"
% date.today().year).decode("utf-8").encode("utf-16be"),
"ANSA":
("© %s ANSA, Tutti i diritti riservati. Testi, foto, grafica non potranno essere pubblicali, riscritti, commercializzati, distribuiti, videotrasmessi, da parte dagli tanti e del terzi in genere, in alcun modo e sotto qualsiasi forma."
% date.today().year).decode("utf-8").encode("utf-16be"),
"SID":
("Alle Rechte für die Wiedergabe, Verwertung und Darstellung reserviert. © %s SID"
% date.today().year).decode("utf-8").encode("utf-16be"),
"dpa":
("Alle Rechte für die Wiedergabe, Verwertung und Darstellung reserviert. © %s dpa"
% date.today().year).decode("utf-8").encode("utf-16be"),
"NU.nl":
("© %s Sanoma Digital The Netherlands B.V. NU - onderdeel van Sanoma Media Netherlands Group"
% date.today().year).decode("utf-8").encode("utf-16be"),
"Reuters_Japanese":
("© Copyright Reuters %s. All rights reserved. ユーザーは、自己の個人的使用及び非商用目的に限り、このサイトにおけるコンテンツの抜粋をダウンロードまたは印刷することができます。ロイターが事前に書面により承認した場合を除き、ロイター・コンテンツを再発行や再配布すること（フレーミングまたは類似の方法による場合を含む）は、明示的に禁止されています。Reutersおよび地球をデザインしたマークは、登録商標であり、全世界のロイター・グループの商標となっています。 "
% date.today().year).decode("utf-8").encode("utf-16be"),
}
for article in data.values():
if article[8] not in sources:
if article[8] in source_names:
source_name = source_names[article[8]]
source_table["name_size_%s" % article[8]] = u32(
len(source_name)) # Size of the source name.
source_table[
"name_offset_%s" %
article[8]] = offset_count() # Offset for the source name.
source_name_copyright[
"source_name_read_%s" %
article[8]] = source_name # Read the source name.
source_name_copyright["padding_source_name_%s" %
article[8]] = u16(
0) # Padding for the source name.
copyright = copyrights[article[8]]
source_table["copyright_size_%s" % article[8]] = u32(
len(copyright)) # Size of the copyright.
source_table[
"copyright_offset_%s" %
article[8]] = offset_count() # Offset for the copyright.
source_name_copyright[
"copyright_read_%s" %
article[8]] = copyright # Read the copyright.
source_name_copyright["padding_copyright_%s" % article[8]] = u16(
0) # Padding for the copyright.
sources.append(article[8])
def make_source_table(header, articles_table, data):
source_table = collections.OrderedDict()
dictionaries.append(source_table)
header["source_offset"] = offset_count() # Offset for the source table.
source_articles = []
"""These are the picture and position values."""
source_nums = {
"AP": [0, 1],
"Reuters": [0, 4],
"AFP": [4, 4],
"AFP_French": [4, 4],
"ANP": [0, 5],
"ANSA": [6, 6],
"dpa": [0, 4],
"SID": [0, 4],
"NU.nl": [0, 5],
"Reuters_Japanese": [0, 4],
}
numbers = 0
numbers_article = 0
for article in data.values():
if article[8] not in source_articles:
source_articles.append(article[8])
source = source_nums[article[8]]
source_table["source_picture_%s" % article[8]] = u8(
source[0]) # Picture for the source.
source_table["source_position_%s" % article[8]] = u8(
source[1]) # Position for the source.
source_table["padding_%s" % article[8]] = u16(0) # Padding.
source_table["pictures_size_%s" % article[8]] = u32(
0) # Size of the source picture.
source_table["pictures_offset_%s" % article[8]] = u32(
0) # Offset for the source picture.
source_table["name_size_%s" % article[8]] = u32(
0) # Size of the source name.
source_table["name_offset_%s" % article[8]] = u32(
0) # Offset for the source name.
source_table["copyright_size_%s" % article[8]] = u32(
0) # Size of the copyright.
source_table["copyright_offset_%s" % article[8]] = u32(
0) # Offset for the copyright.
numbers += 1
for article in data.values():
numbers_article += 1
articles_table["source_%s_number" % numbers_article] = u32(
source_articles.index(article[8])) # Number for the source.
header["source_number"] = u32(
numbers) # Number of entries for the source table.
return source_table
def make_news_bin(mode, console, data, locations_data):
global system, dictionaries, languages, country_code, language_code
source = sources[mode]
if source is None:
print "Could not find %s in sources!"
topics_news = source["topics_news"]
languages = source["languages"]
language_code = source["language_code"]
country_code = source["country_code"]
numbers = 0
if not os.path.exists("newstime"):
os.mkdir("newstime")
for topics in topics_news.values():
newstime = collections.OrderedDict()
for keys in data.keys():
if topics in keys:
numbers += 1
newstime[data[keys][3]] = get_timestamp(1) + u32(numbers)
pickle.dump(
newstime,
open(
"newstime/newstime.%s-%s-%s-%s" %
(str(datetime.now().hour).zfill(2), mode, topics, console),
"wb"))
dictionaries = []
header = make_header(data)
make_wiimenu_articles(header, data)
topics_table = make_topics_table(header, topics_news)
make_timestamps_table(mode, topics_table, topics_news)
articles_table = make_articles_table(mode, locations_data, header, data)
source_table = make_source_table(header, articles_table, data)
locations_table = make_locations_table(header, locations_data)
pictures_table = make_pictures_table(header, data)
make_articles(articles_table, pictures_table, data)
make_topics(topics_table, topics_news)
make_source_name_copyright(source_table, data)
make_locations(locations_data, locations_table)
make_source_pictures(source_table, data)
make_pictures(pictures_table, data)
make_riiconnect24_text()
write_dictionary(mode)
headlines = []
for article in data.values():
if article[3].decode("utf-16be") not in headlines:
headlines.append(article[3].decode("utf-16be") + "\n")
make_news = "".join(headlines)
return make_news
