예제 #1
0
def my_put(resource, item, vpapi):
    ex = vpapi.get(resource, where={"id": item['id']})
    if len(ex['_items']) >= 1:
        #somehow vpapi.put does not work for me, so delete and post
        #vpapi.put(resource,item['id'],item)
        vpapi.delete(resource, item['id'])
    vpapi.post(resource, item)
예제 #2
0
def savevoteevent(self):
  r = vpapi.get('vote-events', where={'identifier':self["identifier"]})
  if not r['_items']:
    #print(self)
    r = vpapi.post("vote-events",self)
  #else:
  #  r = vpapi.patch('vote-events/%s' % r['_items'][0]['id'],self)
    if r['_status'] != 'OK':
        raise Exception(self.name, r)
    else:
        return r
예제 #3
0
def savemotion(self):
    r = vpapi.get('motions', where={'identifiers': {'$elemMatch': self["identifiers"][0]}})
    if not r['_items']:
    #print(self)
        r = vpapi.post("motions",self)
#    else:
#        r = vpapi.put('motions/%s' % r['_items'][0]['id'],self)
    if r['_status'] != 'OK':
        raise Exception(self.name, r)
    else:
        return r
    def test(self):
        last_motion = vpapi.get("votes", page="1")
        pprint.pprint(last_motion["_links"])
        if len(last_motion["_items"]) > 0:
            last_motion_page_text = last_motion["_links"]["last"]["href"]
            index = last_motion_page_text.index("page=") + 5
            last_motion_page = last_motion_page_text[index:]
            pprint.pprint(last_motion_page.encode("utf-8"))
        else:
            last_motion_page = None

        if last_motion_page:
            last_page_motions = vpapi.get("votes", page=last_motion_page)
            last_page_motions_list = []
            for motion in last_page_motions["_items"]:
                last_page_motions_list.append(motion["vote_event_id"])
            print last_page_motions_list[-1]
            # index_start = next(index for (index, d) in enumerate(motions) if d["identifier"] == last_page_motions_list[-1]) + 1
        else:
            index_start = 0
        print index_start
def save(scraped):
    import json

    r = vpapi.get("organizations", where={"identifiers": {"$elemMatch": scraped["identifiers"][0]}})
    if not r["_items"]:
        r = vpapi.post("organizations", scraped)
    else:
        # update by PUT is preferred over PATCH to correctly remove properties that no longer exist now
        existing = r["_items"][0]
        r = vpapi.put("organizations/%s" % existing["id"], scraped)
    if r["_status"] != "OK":
        raise Exception(self.name, resp)
    return r["id"]
    def get_or_create(self, endpoint, item, refresh=False, where_keys=None):
        sort = []
        embed = []
        where = {}
        if where_keys:
            for key in where_keys:
                where[key] = item[key]
        elif endpoint == 'memberships':
            where = {
                'person_id': item['person_id'],
                'organization_id': item['organization_id']
            }
            where['start_date'] = item.get('start_date', {"$exists": False})

            sort = [('start_date', -1)]
        elif endpoint in ('motions', 'speeches'):
            where = {'sources.url': item['sources'][0]['url']}
        elif endpoint == 'vote-events':
            embed = ['votes']
            if 'motion_id' in item:
                where = {'motion_id': item['motion_id']}
            else:
                where = {'start_date': item['start_date']}
        elif endpoint == 'votes':
            where = {
                'vote_event_id': item['vote_event_id'],
                'voter_id': item['voter_id'],
            }
        elif endpoint == 'events':
            where = {'identifier': item['identifier']}
        else:
            where = {
                'identifiers': {'$elemMatch': item['identifiers'][0]}}
        created = False
        resp = vpapi.getfirst(endpoint, where=where, sort=sort)
        if not resp:
            resp = vpapi.post(endpoint, item)
            created = True
            self.log('Created %s' % resp['_links']['self']['href'], DEBUG)
        else:
            pk = resp['id']
            resp = vpapi.put("%s/%s" % (endpoint, pk), item)
            self.log('Updated %s' % resp['_links']['self']['href'], DEBUG)

        if resp['_status'] != 'OK':
            raise Exception(resp)
        if refresh:
            resp = vpapi.get(
                resp['_links']['self']['href'], sort=sort, embed=embed)
        resp['_created'] = created
        return resp
예제 #7
0
def saveperson(scraped):
    import json
    for ident in scraped["identifiers"]:
        if ident["scheme"] == "psp.cz/osoby":
            identifier = ident
            break

    r = vpapi.get('people', where={'identifiers': {'$elemMatch': identifier}})
    if not r['_items']:
        r = vpapi.post('people', scraped)
    else:
        # update by PUT is preferred over PATCH to correctly remove properties that no longer exist now
        existing = r['_items'][0]
        # somehow vpapi.put does not work for me, so delete and post
#        r = vpapi.put('people', existing['id'], scraped)
        vpapi.delete("people", existing['id'])
        r = vpapi.post('people', scraped)
    if r['_status'] != 'OK':
        raise Exception(self.name, resp)
    return r['id']
예제 #8
0
def save_organization(scraped):

    r = vpapi.get('organizations', where={'identifiers': {'$elemMatch': scraped["identifiers"][0]}})
    if not r['_items']:
        r = vpapi.post('organizations', scraped)
        print ("POST " + scraped['id'])
#        outid = r['id']
    else:
        # update by PUT is preferred over PATCH to correctly remove properties that no longer exist now
#        outid = r['_items'][0]['id']
        existing = r['_items'][0]
#        r = vpapi.put('organizations', existing['id'], scraped)
        #somehow vpapi.put does not work for me, so delete and post
        #vpapi.put(resource,item['id'],item)
        vpapi.delete("organizations",existing['id'])
        r = vpapi.post('organizations', scraped)
        print ("PUT " + scraped['id'])
    if r['_status'] != 'OK':
        raise Exception(scraped.name, r)
    return r['id']
    def get_remote_id(self, scheme, identifier):
        key = "%s/%s" % (scheme, identifier)
        if key in self._ids:
            return self._ids[key]

        domain, category = scheme.split('/')
        if category in ('committees', 'parties', 'chamber'):
            endpoint = 'organizations'
        else:
            endpoint = category

        resp = vpapi.get(endpoint, where={
            'identifiers': {
                '$elemMatch': {'scheme': scheme, 'identifier': identifier}
            }
        })

        if resp['_items']:
            item = resp['_items'][0]
            self._ids[key] = item['id']
            return item['id']
예제 #10
0
def savemembership(self):
    r = vpapi.get('memberships',where={'person_id': self["person_id"], 'organization_id': self["organization_id"], "role": "member", "start_date": self["start_date"]})
    if not r['_items']:
        r = vpapi.post("memberships",self)
    else:
        #somehow vpapi.put does not work for me, so delete and post
        update = True
        try:
            if r['_items'][0]["end_date"] == self["end_date"]:
                update = False
                print("not updating: " + r['_items'][0]['id'])
        except:
            nothing = 0
        if update:
            vpapi.delete("memberships",r['_items'][0]['id'])
            self['id'] = r['_items'][0]['id']
            r = vpapi.post('memberships', self)
            print("updating: " + self['id'])
        
            
#        r = vpapi.put('memberships/%s' % r['_items'][0]['id'],self)
            if r['_status'] != 'OK':
                raise Exception(self.name, r)
def saveallmotionsandvoteevents(hl_hlasovani): 
    global test
    organizations = {}
    for row in hl_hlasovani:
        try: 
            organizations[row[1].strip()]
        except:
            organizations[row[1].strip()] = vpapi.get('organizations', where={'identifiers': {'$elemMatch': {"identifier": row[1].strip(), "scheme": "psp.cz/organy"}}}) 
        r_org = organizations[row[1].strip()]
      #r_org = vpapi.get('organizations', where={'identifiers': {'$elemMatch': {"identifier": row[1].strip(), "scheme": "psp.cz/organy"}}})
      
        motion = {
            "id": row[0].strip(),
            "organization_id": r_org["_items"][0]['id'],
            "requirement": guess_majority(row[12],row[11]),
            "result": result2result(row[14].strip()),
            "text": row[15].strip(),
            #'identifiers': [{'identifier': row[0].strip(), 'scheme': 'psp.cz/hlasovani'}]
            "sources": [{'url':"http://www.psp.cz/sqw/hlasy.sqw?g=" + row[0].strip()}]
        }
        print("motion: " + motion['id'])
        if int(motion['id']) > 60536:
            r_motion = savemotion(motion)
      
            #r_motion = vpapi.get('motions', where={'sources': {'$elemMatch': {"identifier": row[0].strip(), "scheme": "psp.cz/hlasovani"}}}) #<-wrong: should be with "sources"
            if r_motion["_status"] == "OK":
                vote_event = {
                    "id": row[0].strip(),
                    "motion_id": r_motion['id'],
                    'identifier': row[0].strip(),
                    #"legislative_session_id": row[2].strip(),  #not implemented in api yet
                    "start_date": vpapi.local_to_utc(scrapeutils.cs2iso(row[5].strip() + "T" + row[6].strip())),
                    "result": result2result(row[14].strip()),
                }
                r_voteevent = savevoteevent(vote_event)
                test[row[0].strip()] = {"id":row[0].strip(),"ve":True}
                logging.info('Motion and vote-event saved: ' + str(r_motion['id']))
예제 #12
0
def saveallmotionsandvoteevents(hl_hlasovani):
    organizations = {}
    for row in hl_hlasovani:
        try:
            organizations[row[1].strip()]
        except:
            organizations[row[1].strip()] = vpapi.get(
                "organizations",
                where={"identifiers": {"$elemMatch": {"identifier": row[1].strip(), "scheme": "psp.cz/organy"}}},
            )
        r_org = organizations[row[1].strip()]
        # r_org = vpapi.get('organizations', where={'identifiers': {'$elemMatch': {"identifier": row[1].strip(), "scheme": "psp.cz/organy"}}})

        motion = {
            "id": row[0].strip(),
            "organization_id": r_org["_items"][0]["id"],
            "requirement": guess_majority(row[12], row[11]),
            "result": result2result(row[14].strip()),
            "text": row[15].strip(),
            #'identifiers': [{'identifier': row[0].strip(), 'scheme': 'psp.cz/hlasovani'}]
            "sources": [{"url": "http://www.psp.cz/sqw/hlasy.sqw?g=" + row[0].strip()}],
        }
        print(motion)
        r_motion = savemotion(motion)

        # r_motion = vpapi.get('motions', where={'sources': {'$elemMatch': {"identifier": row[0].strip(), "scheme": "psp.cz/hlasovani"}}}) #<-wrong: should be with "sources"
        if r_motion["_status"] == "OK":
            vote_event = {
                # "id": row[0].strip(),
                "motion_id": r_motion["id"],
                "identifier": row[0].strip(),
                "legislative_session": row[2].strip(),
                "start_date": scrapeutils.cs2iso(row[5].strip() + " " + row[6].strip()),
                "result": result2result(row[14].strip()),
            }
            r_voteevent = savevoteevent(vote_event)
예제 #13
0
def savemotion(self):
    r = vpapi.get('motions', where={'id': self['id']})
    if not r['_items']:
    #print(self)
        r2 = vpapi.post("motions",self)
예제 #14
0
'''creates people from API
see people-example.json
'''

import vpapi
import json

vpapi.parliament("cz/psp")

people = {}
next = True
page = 1
while next:
    peo = vpapi.get("people",page=page)
    for p in peo["_items"]:
        people[p["id"]] = {
            "id": p["id"],
            "name": p["name"],
            "birth_date": p['birth_date'],
            "gender": p["gender"],
            "sort_name": p["sort_name"],
            "given_name": p["given_name"],
            "identifiers": p["identifiers"],
            "family_name": p["family_name"]
        }
        try:
            p["honorific_prefix"]
        except:
            nothing = None
        else:
            people[p["id"]]["honorific_prefix"] = p["honorific_prefix"]
예제 #15
0
            self['id'] = r['_items'][0]['id']
            r = vpapi.post('memberships', self)
            print("updating: " + self['id'])
        
            
#        r = vpapi.put('memberships/%s' % r['_items'][0]['id'],self)
            if r['_status'] != 'OK':
                raise Exception(self.name, r)

zfile = scrapeutils.download('http://www.psp.cz/eknih/cdrom/opendata/poslanci.zip',zipped=True)
zarazeni = scrapeutils.zipfile2rows(zfile,'zarazeni.unl')

from datetime import datetime
i = 0
for row in zarazeni:
    r_org = vpapi.get('organizations', where={'identifiers': {'$elemMatch': {"identifier": row[1].strip(), "scheme": "psp.cz/organy"}}})
    if len(r_org["_items"]) > 0:
        r_pers = vpapi.get('people', where={'identifiers': {'$elemMatch': {"identifier": row[0].strip(), "scheme": "psp.cz/osoby"}}})
        if len(r_pers["_items"]) > 0:
            membership = {
                "label": "Člen",
                "role": "member",
                "person_id": r_pers["_items"][0]['id'],
                "organization_id": r_org["_items"][0]['id'],
#                "id": str(i),
                "start_date": datetime.strptime(row[3].strip(), '%Y-%m-%d %H').strftime('%Y-%m-%d')
            }
            if row[4].strip() != "":
                membership["end_date"] = datetime.strptime(row[4].strip(), '%Y-%m-%d %H').strftime('%Y-%m-%d')
      
            savemembership(membership)
예제 #16
0
    "onah": {
        "name": "Občanské národní hnutí",
        "color": "#000",
        "position": 30,
        "groups": ["Poslanecký klub Občanského národního hnutí"],
        "abbreviation": "ONAH"
    }
}

# organizations
organizations = {}
next = True
page = 1
while next:
    orgs = vpapi.get("organizations",
                     where={"classification": "political group"},
                     page=page)
    for org in orgs["_items"]:
        organizations[org["id"]] = {"id": org["id"], "name": org["name"]}
    page = page + 1
    try:
        orgs["_links"]["next"]
    except:
        next = False

i = 0
for key in parties:
    party = parties[key]
    party['children'] = []
    party['image'] = key + '.png'
    for keyo in organizations:
예제 #17
0
    return 1
  if vote == 'no':
    return -1
  if vote == 'abstain':
    return -1
  else:
    return 0

answers = {}
groups = {}
mps = {}

vpapi.parliament('sk/nrsr')
for ve in ves:
    print(ve)
    vedb = vpapi.get("vote-events", where={"sources.url":{"$regex":"ID="+ve+"$"}})
    idd = vedb['_items'][0]['id']
    r = vpapi.getall("votes",where={"vote_event_id":idd})
    for row in r:
        try:
            answers[row['voter_id']]
        except:
            answers[row['voter_id']] = {"vote":{}}
        answers[row['voter_id']]['vote'][ve] = vote2vote(row['option'])
        if row['group_id'] is not None:
            try:
                groups[row['group_id']]
            except:
                group = vpapi.get("organizations/"+row['group_id'])
                groups[row['group_id']] = {"name": group['name'].replace('Klub ','')}
                groups[row['group_id']]['slug'] = slugify.slugify(groups[row['group_id']]['name'])
예제 #18
0
osoby = scrapeutils.zipfile2rows(zfile, 'osoby.unl')
poslanec = scrapeutils.zipfile2rows(zfile, 'poslanec.unl')


oosoby = {}
for row in osoby:
    oosoby[row[0].strip()] = row

persons = {}
terms = {}
for row in poslanec:
    oid = row[1].strip()
    try:
        terms[row[4].strip()]
    except Exception:
        r_t = vpapi.get("organizations", where={'identifiers': {'$elemMatch': {"identifier": row[4].strip(), "scheme": "psp.cz/organy"}}})
        for ident in r_t["_items"][0]["identifiers"]:
            if ident["scheme"] == "psp.cz/term":
                terms[row[4].strip()] = ident["identifier"]
    try:
        persons[row[1].strip()]
    except Exception:
        person = {
            "id": row[1].strip(),
            "name": oosoby[oid][3].strip() + " " + oosoby[oid][2].strip(),
            "sort_name": oosoby[oid][2].strip() + ", " + oosoby[oid][3].strip(),
            "family_name": oosoby[oid][2].strip(),
            "given_name": oosoby[oid][3].strip(),
            "birth_date": scrapeutils.cs2iso(oosoby[oid][5].strip()),
            "identifiers": [
                {"identifier": row[0].strip(), "scheme": "psp.cz/poslanec/" + terms[row[4].strip()]},
    cvote_events = {}
    cmotions = {}

#    with open(directory + "/data/votes.csv", "w") as fout:
#        csvw = csv.writer(fout)
#        row = ["vote_event_id","voter_id","option","group_id"]
#        csvw.writerow(row)

    for k in sorted(vote_events):
        print(k + " " + getid(vote_events[k]['sources'][0]['url']))
        with open(directory + "/data/votes.csv", "a") as fout:
            csvw = csv.writer(fout)

            if k > last:
                m_id = vote_events[k]["motion_id"]
                motions[m_id] = vpapi.get("motions",where={"id":m_id})
                cmotions[m_id] = motions[m_id]
                
                vos = vpapi.getall("votes",where={"vote_event_id":k})
                for vote in vos:
#                    votes.append(vote)
                    cvotes.append(vote)
                    try:
                        cpeople[vote["voter_id"]] = people[vote["voter_id"]]
                        corganizations[vote["group_id"]] = organizations[vote["group_id"]]
                        cvote_events[vote["vote_event_id"]] = vote_events[vote["vote_event_id"]]
    #                    m_id = vote_events[vote["vote_event_id"]]["motion_id"]
    #                    cmotions[m_id] = motions[m_id]  
                        
                        vote_event_id = getid(cvote_events[vote["vote_event_id"]]['sources'][0]['url'])
                        voter_id = cpeople[vote["voter_id"]]["identifiers"][0]["identifier"]
예제 #20
0
    "onah": {
        "name": "Občanské národní hnutí",
        "color": "#000",
        "position": 30,
        "groups": ["Poslanecký klub Občanského národního hnutí"],
        "abbreviation": "ONAH"
    }

}

# organizations
organizations = {}
next = True
page = 1
while next:
    orgs = vpapi.get("organizations",where={"classification": "political group"},page=page)
    for org in orgs["_items"]:
        organizations[org["id"]] = {"id": org["id"], "name": org["name"]}
    page = page + 1
    try:
        orgs["_links"]["next"]
    except:
        next = False

i = 0
for key in parties:
    party = parties[key]
    party['children'] = []
    party['image'] = key + '.png'
    for keyo in organizations:
        if organizations[keyo]['name'] in party['groups']:
예제 #21
0
'''

import vpapi
import json

vpapi.parliament("cz/psp")

vote_events = {}
next = True
page = 1

with open('../www/json/issue.json') as data_file:
    issue = json.load(data_file)

for key in issue["vote_events"]:
    rve = vpapi.get("vote-events",where={"identifier":key},embed=["motion"])
    try:
        ve = rve["_items"][0]
        issue["vote_events"][key]['available_vote_event'] = True
        vote_event = {
            "id": ve["id"],
            "motion": {
                "text": ve["motion"]["text"],
                "requirement": ve["motion"]["requirement"],
                "id": ve["motion"]["id"]
            },
            "start_date": ve["start_date"],
            "identifier": ve["identifier"],
            "result": ve["result"],
        }
        next = True
예제 #22
0
        return -1
    if vote == 'abstain':
        return -1
    else:
        return 0


answers = {}
groups = {}
mps = {}

vpapi.parliament('sk/nrsr')
for ve in ves:
    print(ve)
    vedb = vpapi.get("vote-events",
                     where={"sources.url": {
                         "$regex": "ID=" + ve + "$"
                     }})
    idd = vedb['_items'][0]['id']
    r = vpapi.getall("votes", where={"vote_event_id": idd})
    for row in r:
        try:
            answers[row['voter_id']]
        except:
            answers[row['voter_id']] = {"vote": {}}
        answers[row['voter_id']]['vote'][ve] = vote2vote(row['option'])
        if row['group_id'] is not None:
            try:
                groups[row['group_id']]
            except:
                group = vpapi.get("organizations/" + row['group_id'])
                groups[row['group_id']] = {
예제 #23
0
def savevoteevent(self):
  r = vpapi.get('vote-events', where={'identifier':self["identifier"]})
  if not r['_items']:
    #print(self)
    r = vpapi.post("vote-events",self)
예제 #24
0
         for v in votes:
             try:
                 data[p['code'] + '_' + v['voter_id']]
             except:
                 data[p['code'] + '_' + v['voter_id']] = {}
                 data[p['code'] + '_' + v['voter_id']]['votes'] = {}
                 data[p['code'] + '_' + v['voter_id']]['chamber'] = p['code']
                 data[p['code'] + '_' + v['voter_id']]['chamber_name'] = p['name']
                 data[p['code'] + '_' + v['voter_id']]['id'] = v['voter_id']
             data[p['code'] + '_' + v['voter_id']]['votes'][ve['id']] = o2o[v['option']] * int(ve[p['code_csv'] + '_polarity'])
             data[p['code'] + '_' + v['voter_id']]['group_id'] = v['group_id']
 
 os = {}   
 for k in data:
     if data[k]['chamber'] == p['code']:
         mpapi = vpapi.get("people",where={"id":data[k]['id']})
         mp = mpapi["_items"][0]
         data[k]['family_name'] = mp['family_name']
         data[k]['name'] = mp['family_name'] + ' ' + mp['given_name']
         data[k]['given_name'] = mp['given_name']
         try:
             o = os[data[k]["group_id"]]
         except:
             oapi = vpapi.get("organizations",where={"id":data[k]["group_id"]})
             o = oapi["_items"][0]
             os[data[k]["group_id"]] = o
         data[k]['group'] = o['name']
         if data[k]['chamber'] == 'senat':
             data[k]['party_abbreviaton'] = g2g[o['name']]
             if data[k]['id'] == '253':
                 data[k]['picture'] = 'http://senat.cz/images/senatori/' + slugify.slugify(data[k]['family_name']) + slugify.slugify(data[k]['given_name'])[0:1] + '_295.jpg'
예제 #25
0
def savevotes(hl_poslanec):
    votes = {}
    voteevents = {}
    people = {}
    organizations = {}
    terms = {}
    for rowp in hl_poslanec:
        # if rowp[0] == 0: chybne hlasovani v db, viz http://www.psp.cz/sqw/hlasy.sqw?g=58297
        #  try:
        #    terms[hl_hlasovani[i][1].strip()]
        #  except:
        #    r_t = vpapi.get("organizations", where={'identifiers': {'$elemMatch': {"identifier": hl_hlasovani[0][1].strip(), "scheme": "psp.cz/organy"}}})
        #    for ident in r_t["_items"][0]["identifiers"]:
        #      if ident["scheme"] == "psp.cz/term":
        #        terms[hl_hlasovani[0][1].strip()] = ident["identifier"]
        try:
            voteevents[rowp[1].strip()]
        except:
            voteevents[rowp[1].strip()] = vpapi.get("vote-events", where={"identifier": rowp[1].strip()})
        r_voteevent = voteevents[rowp[1].strip()]

        try:
            people[rowp[0].strip()]
        except:
            people[rowp[0].strip()] = vpapi.get(
                "people",
                where={
                    "identifiers": {
                        "$elemMatch": {
                            "identifier": rowp[0].strip(),
                            "scheme": {"$regex": "psp.cz/poslanec/*", "$options": "i"},
                        }
                    }
                },
            )
        r_pers = people[rowp[0].strip()]

        try:
            organizations[r_pers["_items"][0]["id"]]
        except:
            organizations[r_pers["_items"][0]["id"]] = vpapi.get(
                "memberships", where={"person_id": r_pers["_items"][0]["id"]}, embed=["organization"]
            )
        r_org = organizations[r_pers["_items"][0]["id"]]

        for rowo in r_org["_items"]:
            if (
                rowo["organization"]["classification"] == "political group"
                and rowo["start_date"] <= r_voteevent["_items"][0]["start_date"]
            ):
                try:
                    rowo["end_date"]
                except:
                    fine = True
                else:
                    if rowo["end_date"] >= r_voteevent["_items"][0]["start_date"]:
                        fine = True
                    else:
                        fine = False
                    # 9 lines to overcome no python's function "isset" ... )-:
                if fine:
                    organization = rowo["organization"]
                    break
        vote = {
            "voter_id": r_pers["_items"][0]["id"],
            "option": option2option(rowp[2].strip()),
            "group_id": organization["id"],
            "vote_event_id": r_voteevent["_items"][0]["id"],
        }
        try:
            votes[r_voteevent["_items"][0]["id"]]
        except:
            votes[r_voteevent["_items"][0]["id"]] = []
        votes[r_voteevent["_items"][0]["id"]].append(vote.copy())
    #    for k in votes:
    #        vpapi.post("votes",votes[k])
    vpapi.post("votes", votes)
예제 #26
0
'''

import vpapi
import json

vpapi.parliament("cz/psp")

vote_events = {}
next = True
page = 1

with open('../www/json/issue.json') as data_file:
    issue = json.load(data_file)

for key in issue["vote_events"]:
    rve = vpapi.get("vote-events", where={"identifier": key}, embed=["motion"])
    try:
        ve = rve["_items"][0]
        issue["vote_events"][key]['available_vote_event'] = True
        vote_event = {
            "id": ve["id"],
            "motion": {
                "text": ve["motion"]["text"],
                "requirement": ve["motion"]["requirement"],
                "id": ve["motion"]["id"]
            },
            "start_date": ve["start_date"],
            "identifier": ve["identifier"],
            "result": ve["result"],
        }
        next = True
예제 #27
0
                li = td.text.strip().split('\xa0')
                vote = {
                    "vote_event_id":
                    iid,
                    "voter_id":
                    pp2id(" ".join([li[2], li[3]]), vote_event['start_date'],
                          p2id),
                    "option":
                    option2option(li[0]),
                    "group_id":
                    o2id[h2s[j].text.strip()]
                }
                votes.append(vote)
            j += 1

        ex = vpapi.get("motions", where={"id": iid})
        if len(ex['_items']) < 1:
            vpapi.post("motions", motion)
        ex = vpapi.get("vote-events", where={"id": iid})
        if len(ex['_items']) < 1:
            vpapi.post("vote-events", vote_event)
        ex = vpapi.get("votes", where={"vote_event_id": iid})
        if len(ex['_items']) < 1:
            vpapi.post("votes", votes)
        else:
            if int(iid) < 14900:
                break
    except:
        print("XXX:" + iid)
        nothing = 0  # "Zmatečné hlasování"
예제 #28
0
'''creates people from API
see people-example.json
'''

import vpapi
import json

vpapi.parliament("cz/psp")

people = {}
next = True
page = 1
while next:
    peo = vpapi.get("people", page=page)
    for p in peo["_items"]:
        people[p["id"]] = {
            "id": p["id"],
            "name": p["name"],
            "birth_date": p['birth_date'],
            "gender": p["gender"],
            "sort_name": p["sort_name"],
            "given_name": p["given_name"],
            "identifiers": p["identifiers"],
            "family_name": p["family_name"]
        }
        try:
            p["honorific_prefix"]
        except:
            nothing = None
        else:
            people[p["id"]]["honorific_prefix"] = p["honorific_prefix"]
    print('http://www.psp.cz/eknih/cdrom/opendata/hl-'+str(term)+'ps.zip')
    zfile = scrapeutils.download('http://www.psp.cz/eknih/cdrom/opendata/hl-'+str(term)+'ps.zip',zipped=True)
    #hl_hlasovani = scrapeutils.zipfile2rows(zfile,'hl'+str(term)+'s.unl')
    for i in range(1,4):
        try:
            hl_poslanec = scrapeutils.zipfile2rows(zfile,'hl'+str(term)+'h'+str(i)+'.unl')
            #savevotes(hl_poslanec)
            votes = {}
            votesli = []
#            terms = {}
            for rowp in hl_poslanec:

                try:
                    voteevents[rowp[1].strip()]
                except:
                    voteevents[rowp[1].strip()] = vpapi.get('vote-events', where={'identifier': rowp[1].strip()})
                r_voteevent = voteevents[rowp[1].strip()]
              
                try:
                    people[rowp[0].strip()]
                except:
                    people[rowp[0].strip()] = vpapi.get('people', where={"identifiers": {"$elemMatch": {"identifier": rowp[0].strip(), "scheme": {"$regex": "psp.cz/poslanec/*", "$options": "i"} }}})
                r_pers = people[rowp[0].strip()]
                
                try: 
                    organizations[r_pers["_items"][0]["id"]]
                except:
                    organizations[r_pers["_items"][0]["id"]] = vpapi.get('memberships',where={"person_id":r_pers["_items"][0]["id"]},embed=["organization"])   
                r_org = organizations[r_pers["_items"][0]["id"]]
              
                for rowo in r_org["_items"]:
예제 #30
0
        {'name': 'PSP','note':'abbreviation'}
      ],
      'founding_date': scrapeutils.cs2iso(row[6].strip())
    }
    if (row[7].strip() != ''):
      org["dissolution_date"] = scrapeutils.cs2iso(row[7].strip())
    save_organization(org)


# political groups
for row in organy:
  if row[2] == '1':
    org = {
      "name": row[4].strip(),
      'classification': 'political group',
      'id': row[0].strip(),
      'identifiers': [
        {"identifier": row[0].strip(), "scheme": 'psp.cz/organy'}
      ],
      'other_names': [
        {'name': row[3].strip(), 'note':'abbreviation'}
      ],
      'founding_date': scrapeutils.cs2iso(row[6].strip())
    }
    if (row[7].strip() != ''):
      org["dissolution_date"] = scrapeutils.cs2iso(row[7].strip())
    # get parent
    r_parent = vpapi.get('organizations', where={'identifiers': {'$elemMatch': {"identifier": row[1].strip(), "scheme": "psp.cz/organy"}}})
    org["parent_id"] = r_parent["_items"][0]["id"]
    save_organization(org)
예제 #31
0
                        iid += 1
                    gid = re.search('par_2=(\d{1,})',
                                    a.xpath('@href')[0]).group(1).strip()
                    o = re.search('O=(\d{1,})',
                                  a.xpath('@href')[0]).group(1).strip()
                    groups[a.text]["identifiers"][gid] = {
                        "scheme": "senat.cz/" + o,
                        "identifier": gid
                    }

# save it
j = 0
for person in people:
    print(j)
    j += 1
    ex = vpapi.get("people", where={"id": person['id']})
    if len(ex['_items']) < 1:
        vpapi.post("people", person)
#vpapi.post("people",people)
group = {"name": "Senát Parlamentu ČR", "classification": "chamber", "id": "1"}
# some are not available by the algorithm above:
vpapi.post("organizations", group)
group = {
    "name": "Nezařazení",
    "classification": "political group",
    "parent_id": "1",
    "id": str(iid)
}
iid += 1
vpapi.post("organizations", group)
group = {