def get_votes(self):
# org.add_source("http://app.toronto.ca/tmmis/getAdminReport.do?function=prepareMemberVoteReport")
tmpdir = tempfile.mkdtemp()
download_files(tmpdir)
# read through each csv file
files = [f for f in os.listdir(tmpdir)]
for f in files:
name = f.replace('.csv', '')
with open(tmpdir + '/' + f, 'rb') as csvfile:
csvfile = csv.reader(csvfile, delimiter=',')
next(csvfile)
for row in csvfile:
session = self.session
date = row[1].split()[0]
v_type = 'other'
passed = 'Carried' in row[6]
if not 'tie' in row[6]:
yes_count, no_count = row[6].split()[1].split('-')
else:
yes_count, no_count = 1, 1
motion = row[3].replace('\xc4', '').replace('\xc2', '')
vote = Vote(self.jurisdiction.division_name, session, date, motion, v_type, passed, int(yes_count), int(no_count))
vote.vote(name, VOTES[row[5]])
scan_files(tmpdir, f, vote, row)
vote.add_source("http://app.toronto.ca/tmmis/getAdminReport.do")
yield vote
os.remove(tmpdir + '/' + f)
shutil.rmtree(tmpdir)
def get_votes(self):
for page in self.iterpages():
for subject in page.xpath('//div[@class="ContainerPanel"]'):
dates = subject.xpath(".//font[@color='#276598']/b/text()")
motions = [x.strip() for x in subject.xpath(".//div[@style='width:260px; float:left;']/text()")]
votes = subject.xpath(".//div[@style='width:150px; float:right;']")
docket = subject.xpath(".//div[@class='HeaderContent']/b/text()")
docket = filter(lambda x: "docket" in x.lower(), docket)
docket = docket[0] if docket else None
for date, motion, vote in zip(dates, motions, votes):
when = dt.datetime.strptime(date, "%m/%d/%Y")
motion = motion.strip()
if motion == "":
self.warning("Skipping vote.")
continue
v = Vote(
session=self.session,
organization="Boston City Council",
type="other",
passed=False,
date=when.strftime("%Y-%m-%d"),
motion=motion,
yes_count=0,
no_count=0,
)
if docket:
v.set_bill(docket)
yes, no, other = 0, 0, 0
vit = iter(vote.xpath("./div"))
vote = zip(vit, vit, vit)
for who, entry, _ in vote:
how = entry.text
who = who.text
if how == "Y":
v.yes(who)
yes += 1
elif how == "N":
v.no(who)
no += 1
else:
v.other(who)
other += 1
for count in v.vote_counts:
count["count"] = {"yes": yes, "no": no, "other": other}[count["vote_type"]]
v.add_source(DURL, note="root")
yield v
def get_votes(self):
# org.add_source("http://app.toronto.ca/tmmis/getAdminReport.do?function=prepareMemberVoteReport")
tmpdir = tempfile.mkdtemp()
download_files(tmpdir)
# read through each csv file
files = [f for f in os.listdir(tmpdir)]
for f in files:
name = f.replace('.csv', '')
with open(tmpdir + '/' + f, 'rb') as csvfile:
csvfile = csv.reader(csvfile, delimiter=',')
next(csvfile)
for row in csvfile:
session = self.session
date = row[1].split()[0]
v_type = 'other'
passed = 'Carried' in row[6]
if not 'tie' in row[6]:
yes_count, no_count = row[6].split()[1].split('-')
else:
yes_count, no_count = 1, 1
motion = row[3].replace('\xc4', '').replace('\xc2', '')
vote = Vote(self.jurisdiction.division_name, session, date,
motion, v_type, passed, int(yes_count),
int(no_count))
vote.vote(name, VOTES[row[5]])
scan_files(tmpdir, f, vote, row)
vote.add_source(
"http://app.toronto.ca/tmmis/getAdminReport.do")
yield vote
os.remove(tmpdir + '/' + f)
shutil.rmtree(tmpdir)
def get_votes(self):
for page in self.iterpages():
for subject in page.xpath('//div[@class="ContainerPanel"]'):
dates = subject.xpath(".//font[@color='#276598']/b/text()")
motions = [
x.strip() for x in subject.xpath(
".//div[@style='width:260px; float:left;']/text()")
]
votes = subject.xpath(
".//div[@style='width:150px; float:right;']")
docket = subject.xpath(
".//div[@class='HeaderContent']/b/text()")
docket = filter(lambda x: "docket" in x.lower(), docket)
docket = docket[0] if docket else None
for date, motion, vote in zip(dates, motions, votes):
when = dt.datetime.strptime(date, "%m/%d/%Y")
motion = motion.strip()
if motion == "":
self.warning("Skipping vote.")
continue
v = Vote(
session=self.session,
organization="Boston City Council",
type='other',
passed=False,
date=when.strftime("%Y-%m-%d"),
motion=motion,
yes_count=0,
no_count=0,
)
if docket:
v.set_bill(docket)
yes, no, other = 0, 0, 0
vit = iter(vote.xpath("./div"))
vote = zip(vit, vit, vit)
for who, entry, _ in vote:
how = entry.text
who = who.text
if how == 'Y':
v.yes(who)
yes += 1
elif how == 'N':
v.no(who)
no += 1
else:
v.other(who)
other += 1
for count in v.vote_counts:
count['count'] = {
"yes": yes,
"no": no,
"other": other
}[count['vote_type']]
v.add_source(DURL, note='root')
yield v
def migrate_votes(self, state):
spec = {}
if state:
spec['state'] = state
for entry in self.billy_db.votes.find(spec, timeout=False):
#def __init__(self, session, date, type, passed,
# yes_count, no_count, other_count=0,
# chamber=None, **kwargs):
when = dt.datetime.strftime(entry['date'], "%Y-%m-%d")
if entry.get('type') is None:
continue
org_name = entry.get('committee')
if org_name:
# There's a committee tied to this vote.
org_id = entry.get('committee_id')
ocdid = _hot_cache.get(org_id)
else:
# OK. We don't have a committee. We should have a chamber.
# Let's fallback on the COW.
ocdid = _hot_cache.get('{state}-{chamber}'.format(**entry))
org_name = name_cache.get(ocdid)
if ocdid is None or org_name is None:
if entry['chamber'] == 'joint':
print("""
XXX: Vote is marked as joint, but has no committee.
This is likely (totally is) a bug. Please fix the scraper. We'll pass on it in the meantime.
VoteID: %s" % (entry['_id'])""")
continue
raise Exception("""Can't look up the legislature? Something
went wrong internally. The cache might be
wrong for some reason. Look into this.""")
v = Vote(
organization=org_name,
organization_id=ocdid,
motion=entry['motion'],
session=entry['session'],
date=when,
type=entry['type'],
passed=entry['passed'],
yes_count=entry['yes_count'],
no_count=entry['no_count'],
other_count=entry['other_count'],
chamber=entry['chamber'],
#created_at=entry['created_at'],
#updated_at=entry['updated_at'],
)
v.identifiers = [{'scheme': 'openstates',
'identifier': entry['_id']}]
v._openstates_id = entry['_id']
for source in entry['sources']:
v.add_source(**source)
if v.sources == []:
continue # emit warning
for vtype in ['yes', 'no', 'other']:
for voter in entry["%s_votes" % (vtype)]:
id = voter.get('leg_id', None)
hcid = _hot_cache.get(id, None)
getattr(v, vtype)(name=voter['name'], id=hcid)
hcid = _hot_cache.get(entry['bill_id'], None)
bid = hcid
# The bill_id coming off the entry is an ugly OpenStates bigID.
# We need to actually get the bill and use that for this. This
# results in kinda a silly slowdown. Perhaps we can cache titles.
bill_id = entry['bill_id'] # as fallback.
if bid:
if bid in name_cache:
bill_id = name_cache[bid]
# The following is the old code path. However, if the bill didn't
# get converted, we won't have the object in the database anyway,
# so, we'll make a memory/network tradeoff here.
# -- PRT
#
# dbill = db.bills.find_one({"_id": bid})
# if dbill:
# bill_id = dbill['name']
v.set_bill(what=bill_id, id=bid, chamber=entry['chamber'])
self.save_object(v)