コード例 #1
0
ファイル: votes.py プロジェクト: fchagnon/scrapers-ca
  def get_votes(self):
    # org.add_source("http://app.toronto.ca/tmmis/getAdminReport.do?function=prepareMemberVoteReport")

    tmpdir = tempfile.mkdtemp()
    download_files(tmpdir)

    # read through each csv file
    files = [f for f in os.listdir(tmpdir)]
    for f in files:

      name = f.replace('.csv', '')

      with open(tmpdir + '/' + f, 'rb') as csvfile:
        csvfile = csv.reader(csvfile, delimiter=',')
        next(csvfile)

        for row in csvfile:
          session = self.session
          date = row[1].split()[0]
          v_type = 'other'
          passed = 'Carried' in row[6]
          if not 'tie' in row[6]:
            yes_count, no_count = row[6].split()[1].split('-')
          else:
            yes_count, no_count = 1, 1
          motion = row[3].replace('\xc4', '').replace('\xc2', '')
          vote = Vote(self.jurisdiction.division_name, session, date, motion, v_type, passed, int(yes_count), int(no_count))
          vote.vote(name, VOTES[row[5]])
          scan_files(tmpdir, f, vote, row)
          vote.add_source("http://app.toronto.ca/tmmis/getAdminReport.do")
          yield vote
      os.remove(tmpdir + '/' + f)
    shutil.rmtree(tmpdir)
コード例 #2
0
    def get_votes(self):
        for page in self.iterpages():
            for subject in page.xpath('//div[@class="ContainerPanel"]'):
                dates = subject.xpath(".//font[@color='#276598']/b/text()")
                motions = [x.strip() for x in subject.xpath(".//div[@style='width:260px; float:left;']/text()")]
                votes = subject.xpath(".//div[@style='width:150px; float:right;']")
                docket = subject.xpath(".//div[@class='HeaderContent']/b/text()")
                docket = filter(lambda x: "docket" in x.lower(), docket)
                docket = docket[0] if docket else None

                for date, motion, vote in zip(dates, motions, votes):
                    when = dt.datetime.strptime(date, "%m/%d/%Y")
                    motion = motion.strip()

                    if motion == "":
                        self.warning("Skipping vote.")
                        continue

                    v = Vote(
                        session=self.session,
                        organization="Boston City Council",
                        type="other",
                        passed=False,
                        date=when.strftime("%Y-%m-%d"),
                        motion=motion,
                        yes_count=0,
                        no_count=0,
                    )

                    if docket:
                        v.set_bill(docket)

                    yes, no, other = 0, 0, 0

                    vit = iter(vote.xpath("./div"))
                    vote = zip(vit, vit, vit)
                    for who, entry, _ in vote:
                        how = entry.text
                        who = who.text

                        if how == "Y":
                            v.yes(who)
                            yes += 1
                        elif how == "N":
                            v.no(who)
                            no += 1
                        else:
                            v.other(who)
                            other += 1

                    for count in v.vote_counts:
                        count["count"] = {"yes": yes, "no": no, "other": other}[count["vote_type"]]

                    v.add_source(DURL, note="root")
                    yield v
コード例 #3
0
    def get_votes(self):
        # org.add_source("http://app.toronto.ca/tmmis/getAdminReport.do?function=prepareMemberVoteReport")

        tmpdir = tempfile.mkdtemp()
        download_files(tmpdir)

        # read through each csv file
        files = [f for f in os.listdir(tmpdir)]
        for f in files:

            name = f.replace('.csv', '')

            with open(tmpdir + '/' + f, 'rb') as csvfile:
                csvfile = csv.reader(csvfile, delimiter=',')
                next(csvfile)

                for row in csvfile:
                    session = self.session
                    date = row[1].split()[0]
                    v_type = 'other'
                    passed = 'Carried' in row[6]
                    if not 'tie' in row[6]:
                        yes_count, no_count = row[6].split()[1].split('-')
                    else:
                        yes_count, no_count = 1, 1
                    motion = row[3].replace('\xc4', '').replace('\xc2', '')
                    vote = Vote(self.jurisdiction.division_name, session, date,
                                motion, v_type, passed, int(yes_count),
                                int(no_count))
                    vote.vote(name, VOTES[row[5]])
                    scan_files(tmpdir, f, vote, row)
                    vote.add_source(
                        "http://app.toronto.ca/tmmis/getAdminReport.do")
                    yield vote
            os.remove(tmpdir + '/' + f)
        shutil.rmtree(tmpdir)
コード例 #4
0
    def get_votes(self):
        for page in self.iterpages():
            for subject in page.xpath('//div[@class="ContainerPanel"]'):
                dates = subject.xpath(".//font[@color='#276598']/b/text()")
                motions = [
                    x.strip() for x in subject.xpath(
                        ".//div[@style='width:260px; float:left;']/text()")
                ]
                votes = subject.xpath(
                    ".//div[@style='width:150px; float:right;']")
                docket = subject.xpath(
                    ".//div[@class='HeaderContent']/b/text()")
                docket = filter(lambda x: "docket" in x.lower(), docket)
                docket = docket[0] if docket else None

                for date, motion, vote in zip(dates, motions, votes):
                    when = dt.datetime.strptime(date, "%m/%d/%Y")
                    motion = motion.strip()

                    if motion == "":
                        self.warning("Skipping vote.")
                        continue

                    v = Vote(
                        session=self.session,
                        organization="Boston City Council",
                        type='other',
                        passed=False,
                        date=when.strftime("%Y-%m-%d"),
                        motion=motion,
                        yes_count=0,
                        no_count=0,
                    )

                    if docket:
                        v.set_bill(docket)

                    yes, no, other = 0, 0, 0

                    vit = iter(vote.xpath("./div"))
                    vote = zip(vit, vit, vit)
                    for who, entry, _ in vote:
                        how = entry.text
                        who = who.text

                        if how == 'Y':
                            v.yes(who)
                            yes += 1
                        elif how == 'N':
                            v.no(who)
                            no += 1
                        else:
                            v.other(who)
                            other += 1

                    for count in v.vote_counts:
                        count['count'] = {
                            "yes": yes,
                            "no": no,
                            "other": other
                        }[count['vote_type']]

                    v.add_source(DURL, note='root')
                    yield v
コード例 #5
0
ファイル: import_billy.py プロジェクト: finestjava/pupa
    def migrate_votes(self, state):
        spec = {}
        if state:
            spec['state'] = state

        for entry in self.billy_db.votes.find(spec, timeout=False):
            #def __init__(self, session, date, type, passed,
            #             yes_count, no_count, other_count=0,
            #             chamber=None, **kwargs):

            when = dt.datetime.strftime(entry['date'], "%Y-%m-%d")
            if entry.get('type') is None:
                continue

            org_name = entry.get('committee')
            if org_name:
                # There's a committee tied to this vote.
                org_id = entry.get('committee_id')
                ocdid = _hot_cache.get(org_id)
            else:
                # OK. We don't have a committee. We should have a chamber.
                # Let's fallback on the COW.
                ocdid = _hot_cache.get('{state}-{chamber}'.format(**entry))
                org_name = name_cache.get(ocdid)
                if ocdid is None or org_name is None:
                    if entry['chamber'] == 'joint':
                        print("""
    XXX: Vote is marked as joint, but has no committee.
    This is likely (totally is) a bug. Please fix the scraper. We'll pass on it in the meantime.
    VoteID: %s" % (entry['_id'])""")
                        continue

                    raise Exception("""Can't look up the legislature? Something
                                     went wrong internally. The cache might be
                                     wrong for some reason. Look into this.""")

            v = Vote(
                organization=org_name,
                organization_id=ocdid,
                motion=entry['motion'],
                session=entry['session'],
                date=when,
                type=entry['type'],
                passed=entry['passed'],
                yes_count=entry['yes_count'],
                no_count=entry['no_count'],
                other_count=entry['other_count'],
                chamber=entry['chamber'],
                #created_at=entry['created_at'],
                #updated_at=entry['updated_at'],
            )
            v.identifiers = [{'scheme': 'openstates',
                             'identifier': entry['_id']}]
            v._openstates_id = entry['_id']

            for source in entry['sources']:
                v.add_source(**source)

            if v.sources == []:
                continue  # emit warning

            for vtype in ['yes', 'no', 'other']:
                for voter in entry["%s_votes" % (vtype)]:
                    id = voter.get('leg_id', None)
                    hcid = _hot_cache.get(id, None)
                    getattr(v, vtype)(name=voter['name'], id=hcid)

            hcid = _hot_cache.get(entry['bill_id'], None)
            bid = hcid

            # The bill_id coming off the entry is an ugly OpenStates bigID.
            # We need to actually get the bill and use that for this. This
            # results in kinda a silly slowdown. Perhaps we can cache titles.

            bill_id = entry['bill_id']  # as fallback.
            if bid:
                if bid in name_cache:
                    bill_id = name_cache[bid]

                # The following is the old code path. However, if the bill didn't
                # get converted, we won't have the object in the database anyway,
                # so, we'll make a memory/network tradeoff here.
                #                       -- PRT
                #
                # dbill = db.bills.find_one({"_id": bid})
                # if dbill:
                #     bill_id = dbill['name']

            v.set_bill(what=bill_id, id=bid, chamber=entry['chamber'])

            self.save_object(v)