def saveallmotionsandvoteevents(hl_hlasovani): 
    global test
    organizations = {}
    for row in hl_hlasovani:
        try: 
            organizations[row[1].strip()]
        except:
            organizations[row[1].strip()] = vpapi.get('organizations', where={'identifiers': {'$elemMatch': {"identifier": row[1].strip(), "scheme": "psp.cz/organy"}}}) 
        r_org = organizations[row[1].strip()]
      #r_org = vpapi.get('organizations', where={'identifiers': {'$elemMatch': {"identifier": row[1].strip(), "scheme": "psp.cz/organy"}}})
      
        motion = {
            "id": row[0].strip(),
            "organization_id": r_org["_items"][0]['id'],
            "requirement": guess_majority(row[12],row[11]),
            "result": result2result(row[14].strip()),
            "text": row[15].strip(),
            #'identifiers': [{'identifier': row[0].strip(), 'scheme': 'psp.cz/hlasovani'}]
            "sources": [{'url':"http://www.psp.cz/sqw/hlasy.sqw?g=" + row[0].strip()}]
        }
        print("motion: " + motion['id'])
        if int(motion['id']) > 60536:
            r_motion = savemotion(motion)
      
            #r_motion = vpapi.get('motions', where={'sources': {'$elemMatch': {"identifier": row[0].strip(), "scheme": "psp.cz/hlasovani"}}}) #<-wrong: should be with "sources"
            if r_motion["_status"] == "OK":
                vote_event = {
                    "id": row[0].strip(),
                    "motion_id": r_motion['id'],
                    'identifier': row[0].strip(),
                    #"legislative_session_id": row[2].strip(),  #not implemented in api yet
                    "start_date": vpapi.local_to_utc(scrapeutils.cs2iso(row[5].strip() + "T" + row[6].strip())),
                    "result": result2result(row[14].strip()),
                }
                r_voteevent = savevoteevent(vote_event)
                test[row[0].strip()] = {"id":row[0].strip(),"ve":True}
                logging.info('Motion and vote-event saved: ' + str(r_motion['id']))
def saveallmotionsandvoteevents(hl_hlasovani):
    organizations = {}
    for row in hl_hlasovani:
        try:
            organizations[row[1].strip()]
        except:
            organizations[row[1].strip()] = vpapi.get(
                "organizations",
                where={"identifiers": {"$elemMatch": {"identifier": row[1].strip(), "scheme": "psp.cz/organy"}}},
            )
        r_org = organizations[row[1].strip()]
        # r_org = vpapi.get('organizations', where={'identifiers': {'$elemMatch': {"identifier": row[1].strip(), "scheme": "psp.cz/organy"}}})

        motion = {
            "id": row[0].strip(),
            "organization_id": r_org["_items"][0]["id"],
            "requirement": guess_majority(row[12], row[11]),
            "result": result2result(row[14].strip()),
            "text": row[15].strip(),
            #'identifiers': [{'identifier': row[0].strip(), 'scheme': 'psp.cz/hlasovani'}]
            "sources": [{"url": "http://www.psp.cz/sqw/hlasy.sqw?g=" + row[0].strip()}],
        }
        print(motion)
        r_motion = savemotion(motion)

        # r_motion = vpapi.get('motions', where={'sources': {'$elemMatch': {"identifier": row[0].strip(), "scheme": "psp.cz/hlasovani"}}}) #<-wrong: should be with "sources"
        if r_motion["_status"] == "OK":
            vote_event = {
                # "id": row[0].strip(),
                "motion_id": r_motion["id"],
                "identifier": row[0].strip(),
                "legislative_session": row[2].strip(),
                "start_date": scrapeutils.cs2iso(row[5].strip() + " " + row[6].strip()),
                "result": result2result(row[14].strip()),
            }
            r_voteevent = savevoteevent(vote_event)
Beispiel #3
0
     terms[row[4].strip()]
 except Exception:
     r_t = vpapi.get("organizations", where={'identifiers': {'$elemMatch': {"identifier": row[4].strip(), "scheme": "psp.cz/organy"}}})
     for ident in r_t["_items"][0]["identifiers"]:
         if ident["scheme"] == "psp.cz/term":
             terms[row[4].strip()] = ident["identifier"]
 try:
     persons[row[1].strip()]
 except Exception:
     person = {
         "id": row[1].strip(),
         "name": oosoby[oid][3].strip() + " " + oosoby[oid][2].strip(),
         "sort_name": oosoby[oid][2].strip() + ", " + oosoby[oid][3].strip(),
         "family_name": oosoby[oid][2].strip(),
         "given_name": oosoby[oid][3].strip(),
         "birth_date": scrapeutils.cs2iso(oosoby[oid][5].strip()),
         "identifiers": [
             {"identifier": row[0].strip(), "scheme": "psp.cz/poslanec/" + terms[row[4].strip()]},
             {"identifier": row[1].strip(), "scheme": "psp.cz/osoby"}
         ]
     }
     if oosoby[oid][6].strip() == "M":
         person['gender'] = 'male'
     else:
         person['gender'] = 'female'
     if oosoby[oid][1].strip() != "":
         person['honorific_prefix'] = oosoby[oid][1].strip()
     if oosoby[oid][4].strip() != "":
         person['honorific_suffix'] = oosoby[oid][4].strip()
     if oosoby[oid][8].strip() != "":
         person['death_date'] = scrapeutils.cs2iso(oosoby[oid][8].strip())
    print(iid)
    try:
        motion = {
            "text": tds[2].xpath('span//text()')[0],
            "result": result2result(tds[5].xpath('text()')[0]),
            "id": iid
        }
    except:
        motion = {"id": iid}

    vote_event = {
        "result": result2result(tds[5].xpath('text()')[0]),
        "id": iid,
        "motion_id": iid,
        "start_date":
        scrapeutils.cs2iso(tds[3].xpath('text()')[0]) + "T12:00:00"
    }

    url1 = "http://www.senat.cz/xqw/xervlet/pssenat/hlasy?G=" + iid
    domtree1 = html.fromstring(scrapeutils.download(url1))

    tables = domtree1.xpath('//table')
    try:
        quorum = int(
            re.search('BA=(\d{1,})',
                      tables[0].xpath('tr/td')[1].text).group(1).strip())
        present = int(
            re.search('MNO=(\d{1,})',
                      tables[0].xpath('tr/td')[0].text).group(1).strip())

        motion['requirement'] = guess_majority(quorum, present)
# chamber:
for row in organy:
  if row[2] == '11':
    term = row[3][3:]
    org = {
      "name": row[4].strip(),
      'classification': 'chamber',
      'id': row[0].strip(),
      'identifiers': [
        {"identifier": term, 'scheme': 'psp.cz/term'},
        {"identifier": row[0].strip(), "scheme": 'psp.cz/organy'}
      ],
      'other_names': [
        {'name': 'PSP','note':'abbreviation'}
      ],
      'founding_date': scrapeutils.cs2iso(row[6].strip())
    }
    if (row[7].strip() != ''):
      org["dissolution_date"] = scrapeutils.cs2iso(row[7].strip())
    save_organization(org)


# political groups
for row in organy:
  if row[2] == '1':
    org = {
      "name": row[4].strip(),
      'classification': 'political group',
      'id': row[0].strip(),
      'identifiers': [
        {"identifier": row[0].strip(), "scheme": 'psp.cz/organy'}
Beispiel #6
0
table = domtree.xpath('//table')[0]
trs = table.xpath('tr')
for tr in trs:
    tds = tr.xpath('td')
    iid = re.search('G=(\d{1,})', tds[6].xpath('a/@href')[0]).group(1).strip()
    motion = {
        "text": tds[2].xpath('span//text()')[0],
        "result": result2result(tds[5].xpath('text()')[0]),
        "id": iid
    }

    vote_event = {
        "result": result2result(tds[5].xpath('text()')[0]),
        "id": iid,
        "motion_id": iid,
        "start_date": scrapeutils.cs2iso(tds[3].xpath('text()')[0])
    }

    url1 = "http://www.senat.cz/xqw/xervlet/pssenat/hlasy?G=" + iid
    domtree1 = html.fromstring(scrapeutils.download(url1))

    tables = domtree1.xpath('//table')
    quorum = int(
        re.search('BA=(\d{1,})',
                  tables[0].xpath('tr/td')[1].text).group(1).strip())
    present = int(
        re.search('MNO=(\d{1,})',
                  tables[0].xpath('tr/td')[0].text).group(1).strip())

    motion['requirement'] = guess_majority(quorum, present)