def _translate_category(category: str, s: str) -> Tuple[bool, str]:
if ref(category) in _category_map() and s == '0':
return True, _category_map()[ref(
category)] # yield correct name from ref name of category
# if it isn't 0, then it doesn't apply, return given
# if not in the list, return given
return False, s
def __badge_check(title_list):
accepted = list(
map(lambda s: ref(s), [
'general assembly resolution author',
'Historical Resolution Author'
]))
title_list = list(map(lambda s: ref(s), title_list))
for accept in accepted:
if any(accept in s for s in title_list):
return True
return False
def __init__(self,
voter,
post_num,
parsed_ballot,
max_entries=10,
require_all_ranks=False):
self.voter_name = ref(voter)
if not self.is_valid_voter(voter):
raise RuntimeError('voter {} ineligible'.format(voter))
self.ballot = parsed_ballot # internally is [[1, 'title'], [2, 'title']]
ranks, resolution_list = [], []
for internal_list in parsed_ballot:
ranks.append(internal_list[0]) # list of numbers as parsed_ballot
resolution_list.append(
internal_list[1]) # list of resolution titles in parsed_ballot
if max(ranks) > max_entries:
raise RuntimeError('more provided rankings than max')
if duplicates(ranks, excluding=[]):
raise RuntimeError('duplicate entry of same rank')
if duplicates(resolution_list):
raise RuntimeError('same resolution provided more than once')
if set(ranks) != set(range(1, max_entries + 1)) and require_all_ranks:
raise RuntimeError('incomplete, rankings not fully expressed')
self.post_num = post_num
print(f'validated entry {self}')
def join_author_lists_as_set(df):
df = df.copy()
df['Co-authors'] = df['Co-authors'].fillna('')
joined = df[['Author', 'Co-authors']] \
.apply(lambda r: [ref(s) for s in set(','.join(r.values).split(','))], axis=1) \
.explode()
joined = joined[joined != '']
return set(joined.values)
def is_valid_voter(voter_name):
""" Returns true if nation is a GA resolution author or co-author or if has trophy saying it is a GA or UN
resolution author. """
if ref(voter_name) in get_full_author_list(): return True
url = 'https://www.nationstates.net/nation={}'.format(
voter_name.lower().replace(' ', '_'))
title_list = [
image.attrs['title']
for image in BeautifulSoup(requests.get(url).text, 'lxml').select(
'div.trophyline span.trophyrack img')
]
time.sleep(2) # rate limit
return AnnRevEntry.__badge_check(title_list)
def _category_map():
d = {
'Advancement of Industry': 'Environmental Deregulation',
'Civil Rights': 'Mild',
'Human Rights': 'Mild',
'Education and Creativity': 'Artistic',
'Environmental': 'Automotive',
'Free Trade': 'Mild',
'Furtherment of Democracy': 'Mild',
'Global Disarmament': 'Mild',
'Health': 'Healthcare',
'International Security': 'Mild',
'Moral Decency': 'Mild',
'Political Stability': 'Mild',
'Regulation': 'Consumer Protection',
'Gun Control': 'Tighten',
'Social Justice': 'Mild'
}
return {ref(k): v for k, v in d.items()} # force ref name for matching
time.sleep(
2) # sleep 5 seconds __between__ pages, exclude first and last
print('loaded web ballot data')
resolutions = pd.read_csv('../../output/ANNUAL_resolutions.csv')
resolutions['Implemented'] = pd.to_datetime(resolutions['Implemented'],
utc=True)
resolutions['Score'] = 0
resolutions['_lowercase_titles'] = resolutions['Title'].str.lower().str.strip()
print('loaded resolutions data')
# check for puppets
aliases = pd.read_csv('../../db/aliases.csv')
aliases['_joined'] = aliases.apply(
lambda r: [ref(s) for s in set(','.join(r.values).split(','))], axis=1)
voters = [ref(e.voter_name) for e in entry_list] # persist list of voters
# validated 2022-04-04 15.54 cy.par
for alias_list in aliases['_joined'].values:
# determine for all aliases whether the alias voted
results_dict = {a: a in voters for a in alias_list}
if sum(results_dict.values()) > 1:
# if more than one alias voted... tell us
duplicate_names = [k for k, v in results_dict.items() if v]
print('aliases ' + str(duplicate_names) + ' appear more than once!')
print('removing later vote')
# find entries of that player, sort by ballot post number
matching_entries = sorted(
def parse_ga(res_num, council=1):
from src.wa_cacher import Cacher
try:
cacher = Cacher.load()
except FileNotFoundError:
cacher = Cacher() # init new
api_url = 'https://www.nationstates.net/cgi-bin/api.cgi?wa={}&id={}&q=resolution'.format(
council, res_num)
in_cacher = cacher.contains(api_url)
if not in_cacher:
this_response = call_api(api_url)
cacher.update(api_url, this_response)
else:
this_response = cacher.get(api_url)
xml = etree.parse(io.StringIO(this_response))
if not xml.xpath('/WA/RESOLUTION/NAME'):
raise ValueError(
f'resolution number {res_num} is invalid; no such resolution exists'
)
resolution_is_repealed = xml.xpath('/WA/RESOLUTION/REPEALED_BY') != []
resolution_is_a_repeal = xml.xpath(
'/WA/RESOLUTION/REPEALS_COUNCILID') != []
resolution_text = html.unescape(
xml.xpath('/WA/RESOLUTION/DESC')[0].text)
resolution_author = xml.xpath('/WA/RESOLUTION/PROPOSED_BY')[0].text
print(resolution_author)
print(type(resolution_author))
if resolution_author is None or str(resolution_author).strip() == '':
raise RuntimeError('resolution author is empty')
author = capitalise(resolution_author)
resolution = WaPassedResolution(
council=_get_council(council),
resolution_num=res_num,
title=xml.xpath('/WA/RESOLUTION/NAME')[0].text,
implementation=localised(
datetime.utcfromtimestamp(
int(xml.xpath('/WA/RESOLUTION/IMPLEMENTED')[0].text)),
'UTC').astimezone(
timezone('US/Eastern')), # convert to eastern time
chamber=clean_chamber_input(
xml.xpath('/WA/RESOLUTION/COUNCIL')[0].text)[1],
category=capitalise(xml.xpath('/WA/RESOLUTION/CATEGORY')[0].text),
strength=capitalise(
_translate_category(
xml.xpath('/WA/RESOLUTION/CATEGORY')[0].text, # category
xml.xpath('/WA/RESOLUTION/OPTION')[0].text # option
)[1] # get name
),
is_repealed=resolution_is_repealed,
repealed_by=int(xml.xpath('/WA/RESOLUTION/REPEALED_BY')[0].text)
if resolution_is_repealed else None,
is_repeal=resolution_is_a_repeal,
repeals=int(xml.xpath('/WA/RESOLUTION/REPEALS_COUNCILID')[0].text)
if resolution_is_a_repeal else None,
# text and author
text=resolution_text.strip(),
author=author.strip(),
# vote data
votes_for=int(xml.xpath('/WA/RESOLUTION/TOTAL_VOTES_FOR')[0].text),
votes_against=int(
xml.xpath('/WA/RESOLUTION/TOTAL_VOTES_AGAINST')[0].text))
assert resolution.strength != '0', 'resolution {} has strength 0 with category {}'.format(
resolution.title, resolution.category)
# overwrite category if repeal with the repeals field; NS API is broken sometimes for some reason
if resolution_is_a_repeal:
resolution.strength = str(int(
resolution.repeals)) # cast to integer
# check for co-authors
coauth_list = xml.xpath('/WA/RESOLUTION/COAUTHOR/N')
if len(coauth_list) != 0:
print('received from API coauthors: {}'.format(', '.join(
[capitalise(n.text) for n in coauth_list])))
try:
resolution.coauthor0 = capitalise(coauth_list[0].text)
except IndexError:
pass
try:
resolution.coauthor1 = capitalise(coauth_list[1].text)
except IndexError:
pass
try:
resolution.coauthor2 = capitalise(coauth_list[2].text)
except IndexError:
pass
else:
cleaned_resolution_text = resolution_text \
.replace('[i]', '').replace('[/i]', '') \
.replace('[b]', '').replace('[/b]', '') \
.replace('[u]', '').replace('[/u]', '')
coauthor_matches = [
s for s in cleaned_resolution_text.splitlines() if re.search(
r'(Co-?((Author(ed)?:?)|written|writer) ?(by|with)? ?:?)|'
r'(This resolution includes significant contributions made by\s+)',
s, re.IGNORECASE)
]
if len(coauthor_matches) > 0:
coauthor_line = re.sub(
r'Co-?((Author(ed)?:?)|written|writer) ?(by|with)? ?:? ',
repl='',
string=coauthor_matches[0],
flags=re.IGNORECASE)
print(f'\tidentified coauthor line: "{coauthor_line}"')
coauthor_line = coauthor_line \
.replace('[i]', '') \
.replace('[/i]', '') \
.replace('[b]', '') \
.replace('[/b]', '') \
.replace('[u]', '') \
.replace('[/u]', '')
if '[nation' in coauthor_line.lower(
): # scion used the [Nation] tag instead of lower case once
amended_line = re.sub(
r'(?<=\[nation)=(.*?)(?=\])', '',
coauthor_line.lower()) # remove 'noflag' etc
coauthors = re.findall(
r'(?<=\[nation\])(.*?)(?=\[/nation\])',
amended_line.lower())
else:
# this will break with names like "Sch'tz and West Runk'land"
coauthors = re.split(r'(,? and )|(, )', coauthor_line,
re.IGNORECASE)
coauthors = [
i for i in coauthors
if i is not None and i.strip() != 'and'
] # post facto patching...
coauthors = [ref(s).replace('.', '')
for s in coauthors] # cast to reference name
print(f'\tidentified coauthors as {coauthors}')
# pass each co-author in turn
'''
While it could be changed so that the original line's capitalisation is preserved, doing this might
introduce inconsistency in capitalisation of the same nation. Eg '[nation]imperium_anglorum[/nation]' would
be done under capitalisation rules while something provided as 'Imperium ANGLORUM' would be let through.
Because some authors use a ref'd name IN the nation tags, something like [nation]transilia[/nation] cannot
be disentangled from 'Transilia' if the former is proper and the latter is not. A proper-capitalisation
dictionary would be necessary and I am unwilling to download and parse all historical daily dumps for
something this minor.
'''
try:
resolution.coauthor0 = capitalise(coauthors[0])
except IndexError:
pass
try:
resolution.coauthor1 = capitalise(coauthors[1])
except IndexError:
pass
try:
resolution.coauthor2 = capitalise(coauthors[2])
except IndexError:
pass
cacher.save()
return resolution