def my_put(resource, item, vpapi):
    ex = vpapi.get(resource, where={"id": item['id']})
    if len(ex['_items']) >= 1:
        #somehow vpapi.put does not work for me, so delete and post
        #vpapi.put(resource,item['id'],item)
        vpapi.delete(resource, item['id'])
    vpapi.post(resource, item)
    def scrape_chamber(self):
        # Scrapes chambers and Returns the list of chambers with all the information needed for each
        url = "http://www.parliament.am/deputies.php?sel=ful&ord=photo&show_session=5&lang=arm&enc=utf8"
        soup = scrape.download_html_file(url)
        chambers_list = []
        print "\n\tScraping chambers from Armenia's parliament...\n"
        widgets = ['        Progress: ', Percentage(), ' ', Bar(marker='#', left='[', right=']'),
                   ' ', ETA(), " - Processed: ", Counter(), ' items             ']
        pbar = ProgressBar(widgets=widgets)
        all_options = soup.find("select", {"name": "show_session"}).findAll("option")
        for each_option in pbar(all_options):
            identifier = each_option.get('value')
            name = each_option.get_text()
            url = "http://www.parliament.am/deputies.php?lang=arm&sel=&ord=&show_session=" + identifier
            if "100" not in identifier:
                founding_date = self.terms[identifier]["start_date"]
                dissolution_date = self.terms[identifier]["end_date"]
                chamber_json = self.build_organization_doc("chamber", name, identifier, founding_date,
                                                           dissolution_date, url, "", "")

                del chamber_json['contact_details']
                del chamber_json['parent_id']
                if identifier == "5":
                    del chamber_json['dissolution_date']

                existing = vpapi.getfirst("organizations", where={'identifiers': {'$elemMatch': chamber_json['identifiers'][0]}})
                if not existing:
                    resp = vpapi.post("organizations", chamber_json)
                else:
                    resp = vpapi.put("organizations", existing['id'], chamber_json, effective_date=self.effective_date())
                if resp["_status"] != "OK":
                    raise Exception("Invalid status code")
                chambers_list.append(chamber_json)
        print "\n\tScraping completed! \n\tScraped " + str(len(chambers_list)) + " chambers"
        return chambers_list
Beispiel #3
0
 def log_start(self):
     log_item = {
         'status': 'running'
     }
     if settings.get('LOG_FILE'):
         log_item['file'] = settings['LOG_FILE']
     self._log = vpapi.post('logs', log_item)
Beispiel #4
0
	def save(self, update_only=False):
		"""If a compatible membership already exists, update it. Otherwise,
		create a new one. If `update_only` is True, only existing memberships
		are updated, no new one is created.
		Memberships are compatible if their fields `start_date`, `role` and `post`
		are compatible. Field 'end_date' is not checked to allow for later corrections
		of guessed end dates used when a member disappears from a group profile.
		"""
		memberships = vpapi.getall('memberships',
			where={'person_id': self.person_id, 'organization_id': self.organization_id},
			sort='-start_date')
		to_save = self.__dict__.copy()

		id = None
		for existing in memberships:
			if self._merge_values('start_date', to_save, existing) \
					and to_save.get('end_date', '9999-12-31') >= existing.get('start_date', '0001-01-01') \
					and self._merge_values('role', to_save, existing) \
					and self._merge_values('post', to_save, existing):
				id = existing['id']
				self._merge_values('end_date', to_save, existing)
				break
			else:
				to_save = self.__dict__.copy()

		if id:
			resp = vpapi.put('memberships', id, to_save)
		else:
			if update_only: return
			resp = vpapi.post('memberships', self.__dict__)

		if resp['_status'] != 'OK':
			raise Exception(self.name, resp)
    def scrape_chamber(self):
        # Iterates in every parliamentary group json document and
        # returns the list with the json document structure that Visegrad+ API accepts
        print "\n\tScraping chambers from Belarus Lowerhouse parliament..."
        chambers = parser.chambers()
        chambers_list = []
        url = "http://house.gov.by/index.php/,10087,,,,2,,,0.html"
        for chamber in chambers:
            chamber_json = self.build_organization_doc("chamber", chambers[chamber]['name'], chamber,
                                                       chambers[chamber]['start_date'], chambers[chamber]['end_date'],
                                                       url, "", "")
            if chamber == "2":
                del chamber_json['dissolution_date']
            del chamber_json['contact_details']
            del chamber_json['parent_id']

            existing = vpapi.getfirst("organizations", where={'identifiers': {'$elemMatch': chamber_json['identifiers'][0]}})
            if not existing:
                resp = vpapi.post("organizations", chamber_json)
            else:
                # update by PUT is preferred over PATCH to correctly remove properties that no longer exist now
                resp = vpapi.put("organizations", existing['id'], chamber_json, effective_date=self.effective_date())
            if resp["_status"] != "OK":
                raise Exception("Invalid status code")
            chambers_list.append(chamber_json)
        print "\n\tScraping completed! \n\tScraped " + str(len(chambers_list)) + " chambers"
        return chambers_list
def savevoteevent(self):
    # r = vpapi.get('vote-events', where={'identifier':self["identifier"]})
    # if not r['_items']:
    r = vpapi.post("vote-events", self)
    # else:
    #  r = vpapi.patch('vote-events/%s' % r['_items'][0]['id'],self)
    if r["_status"] != "OK":
        raise Exception(self.name, r)
    else:
        return r
def savemotion(self):
    # r = vpapi.get('motions', where={'identifiers': {'$elemMatch': self["identifiers"][0]}})
    #   if not r['_items']:
    r = vpapi.post("motions", self)
    #   else:
    #       r = vpapi.put('motions/%s' % r['_items'][0]['id'],self)
    if r["_status"] != "OK":
        raise Exception(self.name, r)
    else:
        return r
def save_organization(scraped):

    r = vpapi.get('organizations', where={'identifiers': {'$elemMatch': scraped["identifiers"][0]}})
    if not r['_items']:
        r = vpapi.post('organizations', scraped)
        print ("POST " + scraped['id'])
#        outid = r['id']
    else:
        # update by PUT is preferred over PATCH to correctly remove properties that no longer exist now
#        outid = r['_items'][0]['id']
        existing = r['_items'][0]
#        r = vpapi.put('organizations', existing['id'], scraped)
        #somehow vpapi.put does not work for me, so delete and post
        #vpapi.put(resource,item['id'],item)
        vpapi.delete("organizations",existing['id'])
        r = vpapi.post('organizations', scraped)
        print ("PUT " + scraped['id'])
    if r['_status'] != 'OK':
        raise Exception(scraped.name, r)
    return r['id']
Beispiel #9
0
def saveperson(scraped):
    import json
    for ident in scraped["identifiers"]:
        if ident["scheme"] == "psp.cz/osoby":
            identifier = ident
            break

    r = vpapi.get('people', where={'identifiers': {'$elemMatch': identifier}})
    if not r['_items']:
        r = vpapi.post('people', scraped)
    else:
        # update by PUT is preferred over PATCH to correctly remove properties that no longer exist now
        existing = r['_items'][0]
        # somehow vpapi.put does not work for me, so delete and post
#        r = vpapi.put('people', existing['id'], scraped)
        vpapi.delete("people", existing['id'])
        r = vpapi.post('people', scraped)
    if r['_status'] != 'OK':
        raise Exception(self.name, resp)
    return r['id']
Beispiel #10
0
	def save(self):
		scraped = self.__dict__
		existing = vpapi.getfirst('people', where={'identifiers': {'$elemMatch': self.identifiers[0]}})
		if not existing:
			resp = vpapi.post('people', scraped)
		else:
			# update by PUT is preferred over PATCH to correctly remove properties that no longer exist now
			resp = vpapi.put('people', existing['id'], scraped, effective_date=effective_date)

		if resp['_status'] != 'OK':
			raise Exception(self.name, resp)
		return resp['id']
def save(scraped):
    import json

    r = vpapi.get("organizations", where={"identifiers": {"$elemMatch": scraped["identifiers"][0]}})
    if not r["_items"]:
        r = vpapi.post("organizations", scraped)
    else:
        # update by PUT is preferred over PATCH to correctly remove properties that no longer exist now
        existing = r["_items"][0]
        r = vpapi.put("organizations/%s" % existing["id"], scraped)
    if r["_status"] != "OK":
        raise Exception(self.name, resp)
    return r["id"]
    def get_or_create(self, endpoint, item, refresh=False, where_keys=None):
        sort = []
        embed = []
        where = {}
        if where_keys:
            for key in where_keys:
                where[key] = item[key]
        elif endpoint == 'memberships':
            where = {
                'person_id': item['person_id'],
                'organization_id': item['organization_id']
            }
            where['start_date'] = item.get('start_date', {"$exists": False})

            sort = [('start_date', -1)]
        elif endpoint in ('motions', 'speeches'):
            where = {'sources.url': item['sources'][0]['url']}
        elif endpoint == 'vote-events':
            embed = ['votes']
            if 'motion_id' in item:
                where = {'motion_id': item['motion_id']}
            else:
                where = {'start_date': item['start_date']}
        elif endpoint == 'votes':
            where = {
                'vote_event_id': item['vote_event_id'],
                'voter_id': item['voter_id'],
            }
        elif endpoint == 'events':
            where = {'identifier': item['identifier']}
        else:
            where = {
                'identifiers': {'$elemMatch': item['identifiers'][0]}}
        created = False
        resp = vpapi.getfirst(endpoint, where=where, sort=sort)
        if not resp:
            resp = vpapi.post(endpoint, item)
            created = True
            self.log('Created %s' % resp['_links']['self']['href'], DEBUG)
        else:
            pk = resp['id']
            resp = vpapi.put("%s/%s" % (endpoint, pk), item)
            self.log('Updated %s' % resp['_links']['self']['href'], DEBUG)

        if resp['_status'] != 'OK':
            raise Exception(resp)
        if refresh:
            resp = vpapi.get(
                resp['_links']['self']['href'], sort=sort, embed=embed)
        resp['_created'] = created
        return resp
Beispiel #13
0
def savemembership(self):
    r = vpapi.get('memberships',where={'person_id': self["person_id"], 'organization_id': self["organization_id"], "role": "member", "start_date": self["start_date"]})
    if not r['_items']:
        r = vpapi.post("memberships",self)
    else:
        #somehow vpapi.put does not work for me, so delete and post
        update = True
        try:
            if r['_items'][0]["end_date"] == self["end_date"]:
                update = False
                print("not updating: " + r['_items'][0]['id'])
        except:
            nothing = 0
        if update:
            vpapi.delete("memberships",r['_items'][0]['id'])
            self['id'] = r['_items'][0]['id']
            r = vpapi.post('memberships', self)
            print("updating: " + self['id'])
        
            
#        r = vpapi.put('memberships/%s' % r['_items'][0]['id'],self)
            if r['_status'] != 'OK':
                raise Exception(self.name, r)
Beispiel #14
0
def get_or_create(resource, item, key=None):
	"""Unless the item already exists in the resource (identified by
	`key` fields) create it. Return id of the item and a bool whether
	the item was newly created or not. If key is not given, all fields
	of the item are used as a key.
	"""
	if key is None:
		key = item.keys()
	query = {field: item[field] for field in key}
	existing = vpapi.getfirst(resource, where=query)
	if existing:
		return existing['id'], False
	resp = vpapi.post(resource, item)
	return resp['id'], True
    def scrape_chamber(self):
        # Scrapes chambers and Returns the list of chambers with all the information needed for each
        url = "http://www.parlament.md/Parlamentarismul%C3%AEnRepublicaMoldova/" \
              "Istorie%C8%99ievolu%C8%9Bie/tabid/96/language/ro-RO/Default.aspx"
        chambers_to_fix = {"XII": "12", "XIII": "13", "XIV": "14", "XV": "15", "XVI": "16", "XVII": "17",
                           "XVIII": "18", "XIX": "19", "XX": "20"}
        chambers = []
        soup = scrape.download_html_file(url)
        print "\n\tScraping chambers from Moldova's parliament..."
        widgets = ['        Progress: ', Percentage(), ' ', Bar(marker='#', left='[', right=']'),
                   ' ', ETA(), " - Processed: ", Counter(), ' items             ']
        pbar = ProgressBar(widgets=widgets)
        for each_a in pbar(soup.find('div', {"class": "LocalizedContent"}).findAll('a')):
            name = each_a.get_text().strip()
            if name != "":
                url = "http://www.parlament.md" + each_a.get('href')
                if "(" in name:
                    chamber_roman = name[name.index('X'):name.index('(')].replace('-a', "").strip()
                    chamber_identifier = chambers_to_fix[chamber_roman]
                    founding_date = self.terms[chamber_identifier]['start_date']
                    dissolution_date = self.terms[chamber_identifier]['end_date']
                else:
                    chamber_roman = name[-6:len(name)-3].strip()
                    chamber_identifier = chambers_to_fix[chamber_roman]
                    founding_date = self.terms[chamber_identifier]['start_date']
                    dissolution_date = self.terms[chamber_identifier]['end_date']

                chamber_json = self.build_organization_doc("chamber", name, chamber_identifier, founding_date,
                                                           dissolution_date, url, "", "")

                del chamber_json['contact_details']
                del chamber_json['parent_id']
                if chamber_identifier == "20":
                    del chamber_json['dissolution_date']

                existing = vpapi.getfirst("organizations", where={'identifiers': {'$elemMatch': chamber_json['identifiers'][0]}})
                if not existing:
                    resp = vpapi.post("organizations", chamber_json)
                else:
                    # update by PUT is preferred over PATCH to correctly remove properties that no longer exist now
                    resp = vpapi.put("organizations", existing['id'], chamber_json, effective_date=self.effective_date())
                if resp["_status"] != "OK":
                    raise Exception("Invalid status code")
                chambers.append(chamber_json)
        print "\n\tScraping completed! \n\tScraped " + str(len(chambers)) + " chambers"
        return chambers
def savemotion(self):
    r = vpapi.get('motions', where={'id': self['id']})
    if not r['_items']:
    #print(self)
        r2 = vpapi.post("motions",self)
Beispiel #17
0
def main():
	# read command-line arguments
	ap = argparse.ArgumentParser('Scrapes data from Slovak parliament website http://nrsr.sk')
	ap.add_argument('--people', choices=['initial', 'recent', 'none'], default='recent', help='scrape of people, organizations and memberships')
	ap.add_argument('--votes', choices=['initial', 'recent', 'none'], default='recent', help='scrape of motions and votes')
	ap.add_argument('--debates', choices=['initial', 'recent', 'none'], default='recent', help='scrape of speeches from debates')
	ap.add_argument('--term', help='term to scrape recent data from; current term is used when omitted')
	args = ap.parse_args()

	# set-up logging to a local file
	if not os.path.exists(LOGS_DIR):
		os.makedirs(LOGS_DIR)
	logname = datetime.utcnow().strftime('%Y-%m-%d-%H%M%S') + '.log'
	logname = os.path.join(LOGS_DIR, logname)
	logname = os.path.abspath(logname)
	logging.basicConfig(level=logging.DEBUG, format='%(message)s', handlers=[logging.FileHandler(logname, 'w', 'utf-8')])
	logging.getLogger('requests').setLevel(logging.ERROR)

	logging.info('Started')
	try:
		# set-up the API access
		vpapi.parliament('sk/nrsr')
		vpapi.timezone('Europe/Bratislava')
		with open(os.path.join(CONF_DIR, 'private.json'), encoding='utf8') as f:
			creds = json.load(f)
		vpapi.authorize(creds['api_user'], creds['password'])

		# indicate that the scraper has started
		db_log = vpapi.post('logs', {'status': 'running', 'file': logname, 'params': args.__dict__})

		# clear cached source files
		if scrapeutils.USE_WEBCACHE:
			logging.info('Clearing cached files')
			scrapeutils.clear_cache()

		# test parser functions
		logging.info('Testing parser functions')
		out = io.StringIO()
		suite = unittest.TestLoader().loadTestsFromModule(sys.modules['test'])
		result = unittest.TextTestRunner(stream=out).run(suite)
		logging.info(out.getvalue())
		if result.errors or result.failures:
			raise RuntimeError('Unit tests of parser functions failed, update canceled.')

		if args.people == 'initial':
			# initial scrape of all history of people and organizations
			logging.info('Initial scrape - deleting people, organizations and memberships')
			vpapi.delete('memberships')
			vpapi.delete('organizations')
			vpapi.delete('people')
			for term in sorted(parse.terms.keys()):
				scrape_people(term)

		elif args.people == 'recent':
			# incremental scrape of people and organizations since the last scrape
			term = args.term or parse.current_term()
			if term not in parse.terms:
				raise Exception('Unknown term `%s`. Scrape canceled. Add it to the terms list in parse.py an rerun for the recently finished term once more.' % term)
			scrape_people(term)

		terms_with_old_debates = ('1', '2', '3', '4')
		if args.debates == 'initial':
			# initial scrape of debates from all terms
			logging.info('Initial scrape - deleting speeches and events')
			vpapi.delete('speeches')
			vpapi.delete('events')
			# newer terms are scraped first to get full names of unknown speakers
			for term in sorted(parse.terms.keys()):
				if term in terms_with_old_debates: continue
				scrape_new_debates(term)
			for term in terms_with_old_debates:
				scrape_old_debates(term)

		elif args.debates == 'recent':
			# incremental scrape of debates since the last scrape
			term = args.term or parse.current_term()
			if term not in parse.terms:
				raise Exception('Unknown term `%s`. Scrape canceled. Add it to the terms list in parse.py an rerun once more.' % term)
			if term in terms_with_old_debates:
				scrape_old_debates(term)
			else:
				scrape_new_debates(term)

		if args.votes == 'initial':
			# initial scrape of votes from all terms
			logging.info('Initial scrape - deleting votes, vote-events and motions')
			vpapi.delete('votes')
			vpapi.delete('vote-events')
			vpapi.delete('motions')
			for term in sorted(parse.terms.keys()):
				scrape_motions(term)

		elif args.votes == 'recent':
			# incremental scrape of votes since the last scrape
			term = args.term or parse.current_term()
			if term not in parse.terms:
				raise Exception('Unknown term `%s`. Scrape canceled. Add it to the terms list in parse.py an rerun once more.' % term)
			scrape_motions(term)

		status = 'finished'

	except BaseException as e:
		logging.critical(e, exc_info=True)
		if hasattr(e, 'response') and hasattr(e.response, '_content'):
			logging.critical(e.response._content.decode('utf-8'))
		status = 'interrupted' if isinstance(e, KeyboardInterrupt) else 'failed'

		# output to console to provoke an e-mail from Cron
		print('Scraping of parliament sk/nrsr failed, see\n\n' + logname + '\n\nfor details.')

	finally:
		logging.info(status.capitalize())
		if 'db_log' in locals():
			vpapi.patch('logs', db_log['id'], {'status': status})
Beispiel #18
0
    def export_speeches(self):
        speeches = self.load_json('speeches')
        people = {}
        prefix_regex = re.compile(
            ur'(pred\u015bedavaju\u0107i )|(pred\u015bednik )|\
(generalni sekretar )', re.U)

        for p in vpapi.getall('people'):
            name = self.normalize_name(p['name'])
            people[name] = p['id']

        for speech in speeches:
            session_id = speech.get('event_id')
            speech['event_id'] = self.events_ids[session_id]
            url = speech['sources'][0]['url']
            if url.endswith('.pdf'):
                parsed_speeches = self.download_pdf(url)
                for n, s in enumerate(parsed_speeches):
                    text_speech = speech.copy()
                    text_speech['text'] = s['text']
                    text_speech['position'] = n + 1
                    text_speech['type'] = 'speech'

                    creator = self.normalize_name(s['creator'])
                    creator = prefix_regex.sub('', creator)

                    if creator in people:
                        text_speech['creator_id'] = people[creator]
                    else:
                        creator_id = None

                        for name in people:
                            if name in creator:
                                creator_id = people[name]
                                break

                        if creator_id is None:
                            resp = vpapi.getfirst('people',
                                                  where={
                                                      'name': {
                                                          '$regex':
                                                          s['creator'],
                                                          'options': 'i'
                                                      }
                                                  })
                            if resp is None:
                                self.log(
                                    'Person "%(creator)s" not found. \
Creating one' % s, WARNING)
                                item = {
                                    'name': s['creator'],
                                    'sources': text_speech['sources']
                                }
                                resp = vpapi.post('people', item)
                            creator_id = resp['id']

                        people[creator] = creator_id
                        text_speech['creator_id'] = creator_id

                    self.get_or_create('speeches',
                                       text_speech,
                                       where_keys=['event_id', 'position'])
            else:
                self.get_or_create('speeches', speech)
Beispiel #19
0
                                    a.xpath('@href')[0]).group(1).strip()
                    o = re.search('O=(\d{1,})',
                                  a.xpath('@href')[0]).group(1).strip()
                    groups[a.text]["identifiers"][gid] = {
                        "scheme": "senat.cz/" + o,
                        "identifier": gid
                    }

# save it
j = 0
for person in people:
    print(j)
    j += 1
    ex = vpapi.get("people", where={"id": person['id']})
    if len(ex['_items']) < 1:
        vpapi.post("people", person)
#vpapi.post("people",people)
group = {"name": "Senát Parlamentu ČR", "classification": "chamber", "id": "1"}
# some are not available by the algorithm above:
vpapi.post("organizations", group)
group = {
    "name": "Nezařazení",
    "classification": "political group",
    "parent_id": "1",
    "id": str(iid)
}
iid += 1
vpapi.post("organizations", group)
group = {
    "name": "Senátorský klub Zelení - nezávislí",
    "classification": "political group",
                test[row[0].strip()] = {"id":row[0].strip(),"ve":True}
                logging.info('Motion and vote-event saved: ' + str(r_motion['id']))



# set-up logging to a local file
if not os.path.exists(LOGS_DIR):
	os.makedirs(LOGS_DIR)
logname = datetime.utcnow().strftime('%Y-%m-%d-%H%M%S') + '.log'
logname = os.path.join(LOGS_DIR, logname)
logname = os.path.abspath(logname)
logging.basicConfig(level=logging.DEBUG, format='%(message)s', handlers=[logging.FileHandler(logname, 'w', 'utf-8')])
logging.getLogger('requests').setLevel(logging.ERROR)

logging.info('Started')
db_log = vpapi.post('logs', {'status': 'running', 'file': logname, 'params': []})
            

terms = [1993, 1996, 1998, 2002, 2006, 2010, 2013]
terms = [2013]
test = {}
#terms = [2010]
for term in terms:
    zfile = scrapeutils.download('http://www.psp.cz/eknih/cdrom/opendata/hl-'+str(term)+'ps.zip',zipped=True)
    hl_hlasovani = scrapeutils.zipfile2rows(zfile,'hl'+str(term)+'s.unl')
    saveallmotionsandvoteevents(hl_hlasovani)



#j = 0
#last_ve_id = 0
    def vote_events(self):
        print "\n\n\tScraping Motions and Vote Events data from Ukraine's parliament..."
        vote_events = parser.vote_events_list()
        index_vote_events = self.get_index("vote-events", "-start_date", vote_events)
        index_motions = self.get_index("motions", "-date", vote_events)
        index = min(index_vote_events, index_motions)
        voting_events = []
        motions = []
        if len(vote_events) > 0:
            print "\n\n\tPosting Motions and Vote events data to the Visegrad+ API from Ukraine's parliament..."
            if len(vote_events[index:]) > 0:
                widgets = [
                    "        Progress: ",
                    Percentage(),
                    " ",
                    Bar(marker="#", left="[", right="]"),
                    " ",
                    ETA(),
                    " - Processed: ",
                    Counter(),
                    " events             ",
                ]
                pbar = ProgressBar(widgets=widgets)
                for motion in pbar(vote_events[index:]):
                    json_motion = self.build_json_motion(
                        motion["date"][:19],
                        motion["sources"][0]["url"],
                        motion["id"],
                        motion["legislative_session_id"],
                        motion["organization_id"],
                        motion["text"],
                        motion["result"],
                    )
                    motions.append(json_motion)
                    existing = vpapi.getfirst("motions", where={"identifier": json_motion["identifier"]})
                    if not existing:
                        vpapi.post("motions", json_motion)
                    else:
                        continue

                    json_vote_event = self.build_vote_event_json(
                        motion["date"][:19],
                        motion["legislative_session_id"],
                        motion["id"],
                        motion["organization_id"],
                        motion["result"],
                        motion["counts"],
                    )
                    voting_events.append(json_vote_event)
                    existing1 = vpapi.getfirst("vote-events", where={"id": json_vote_event["id"]})
                    if not existing1:
                        vpapi.post("vote-events", json_vote_event)
                    else:
                        continue
                print "\n\tFinished posting motions and vote events data."
                print "\tScraped %s motions and vote events" % str(len(vote_events[index:]) * 2)
            else:
                print "\n\tThere are no new motion and vote events data."
        else:
            print "\n\tThere are no new motions or vote events."
        return motions, voting_events
def savevoteevent(self):
  r = vpapi.get('vote-events', where={'identifier':self["identifier"]})
  if not r['_items']:
    #print(self)
    r = vpapi.post("vote-events",self)
Beispiel #23
0
def scrape_new_debates(term):
	"""Scrape and save speeches from debates of the given term, one
	of those newer terms where transcripts of debates are published
	in parts assigned to individual speakers.

	Returns number of scraped speeches.
	"""
	debate_part_kinds = {
		'Uvádzajúci uvádza bod': 'speech',
		'Vstup predsedajúceho': 'speech',
		'Vystúpenie spoločného spravodajcu': 'speech',
		'Vystúpenie': 'speech',
		'Vystúpenie v rozprave': 'speech',
		'Vystúpenie s faktickou poznámkou': 'speech',
		'Vystúpenie s procedurálnym návrhom': 'speech',
		'Prednesenie otázky': 'question',
		'Zodpovedanie otázky': 'answer',
		'Doplňujúca otázka / reakcia zadávajúceho': 'question',
		'Prednesenie interpelácie': 'question',
		'Odpoveď na interpeláciu': 'answer',
		'scene': 'scene'
	}

	def insert_speech(kind):
		"""Insert a speech entity for the given debate part kind
		and data from parent scope variables and update end date
		of the corresponding session and sitting. Delete `text`
		variable."""
		nonlocal text, last_speech_enddatetime
		if not text: return
		speech = {
			'text': text.strip().replace('[', '(').replace(']', ')'),
			'date': start_datetime,
			'type': debate_part_kinds.get(kind, 'speech'),
			'position': len(speeches) + 1,
			'event_id': sitting_id,
			'sources' : [{
				'url': dpart_url,
				'note': 'Prepis časti debaty na webe NRSR'
			}]
		}
		if dpart_video:
			speech['video'] = dpart_video
		if kind != 'scene':
			speech['creator_id'] = speaker_id
			speech['attribution_text'] = attribution.strip()
		speeches.append(speech)
		text = ''

		if end_datetime > session_end_date:
			vpapi.patch('events', session_id, {'end_date': end_datetime})
		if end_datetime > sitting_end_date:
			vpapi.patch('events', sitting_id, {'end_date': end_datetime})
		last_speech_enddatetime = datetime.strptime(end_datetime, '%Y-%m-%dT%H:%M:%S')

	logging.info('Scraping debates of term `%s`' % term)
	chamber_id = get_chamber_id(term)

	# prepare mapping from MP's name to id
	people = vpapi.getall('people', projection={'name': 1})
	mps = {mp['name']: mp['id'] for mp in people}

	# load name corrections
	with open(os.path.join(CONF_DIR, 'name_corrections.json'), encoding='utf8') as f:
		name_corrections = json.load(f)

	# scraping will start since the most recent sitting start date
	last_sitting = vpapi.getfirst('events',
		where={'type': 'sitting', 'organization_id': chamber_id},
		sort='-start_date')
	since_date = last_sitting['start_date'][:10] if last_sitting else None

	# scrape list of debate parts
	debate_parts = parse.new_debates_list(term, since_date)

	speech_count = 0
	session_name = ''
	speeches = []
	for dp in debate_parts['_items']:
		# stop at very recent debate parts (may be incomplete)
		start_datetime = sk_to_utc('%s %s' % (dp['dátum'], dp['trvanie']['od']))
		sd = datetime.strptime(start_datetime, '%Y-%m-%dT%H:%M:%S')
		if datetime.utcnow() - sd < timedelta(days=5):
			break

		# skip already scraped debate parts
		existing = vpapi.getfirst('speeches', where={'sources.url': dp['prepis']['url']})
		if existing: continue

		logging.info('Scraping debate part %s %s-%s (id=%s)' %
			(dp['dátum'], dp['trvanie']['od'], dp['trvanie']['do'], dp['prepis']['id']))
		dpart = parse.debate_of_terms56(dp['prepis']['id'])
		if not dpart['riadky']: continue

		end_datetime = sk_to_utc('%s %s' % (dp['dátum'], dp['trvanie']['do']))
		dpart_kind = dp['druh']
		dpart_url = dp['prepis']['url']
		dpart_video = dp['video']['url'] if 'video' in dp else None

		if not session_name.startswith('%s. ' % dp['schôdza']):
			# create new session event
			session_name = '%s. schôdza' % dp['schôdza']
			session = {
				'name': session_name,
				'identifier': dp['schôdza'],
				'organization_id': chamber_id,
				'type': 'session',
				'start_date': start_datetime,
				'end_date': end_datetime,
			}
			key = ('organization_id', 'type', 'identifier')
			session_id, _ = get_or_create('events', session, key)
			session_end_date = end_datetime

			# find the last moment of the last sitting of this session
			session_last_sitting = vpapi.getfirst('events',
				where={'type': 'sitting', 'parent_id': session_id},
				sort='-start_date')
			if session_last_sitting:
				last_speech_enddatetime = datetime.strptime(session_last_sitting['end_date'], '%Y-%m-%dT%H:%M:%S')
				sitting_identifier = session_last_sitting['identifier']
				sitting_id = session_last_sitting['id']
				sitting_end_date = session_last_sitting['end_date']
			else:
				last_speech_enddatetime = datetime.min
				sitting_identifier = '0'

		if sd - last_speech_enddatetime > timedelta(hours=5):
			# create new sitting event
			sitting_identifier = str(int(sitting_identifier) + 1)
			sitting_name = '%s. deň rokovania, %s' % (sitting_identifier, dp['dátum'])
			sitting = {
				'name': sitting_name,
				'identifier': sitting_identifier,
				'organization_id': chamber_id,
				'type': 'sitting',
				'start_date': start_datetime,
				'end_date': end_datetime,
				'parent_id': session_id,
			}
			key = ('parent_id', 'type', 'identifier')
			sitting_id, _ = get_or_create('events', sitting, key)
			sitting_end_date = end_datetime

			# save speeches of the previous sitting
			if len(speeches) > 0:
				vpapi.post('speeches', speeches)
				speech_count += len(speeches)
			if dp != debate_parts['_items'][0]:
				logging.info('Scraped %s speeches from previous sitting' % len(speeches))
			speeches = []

		# add the first speaker name that is sometimes missing
		first_speaker = '<strong>%s, %s</strong>' % (dp['osoba']['meno'], dp['osoba']['funkcia'])
		dpart['riadky'].insert(0, first_speaker)

		# extract speeches from the debate part
		text = ''
		within_scene = False
		for par in dpart['riadky']:
			if not par: continue
			par = par.replace('\n', ' ').strip()

			# skip eventual speech number
			if re.match('^(\d+)\.$', par): continue

			# convert brackets to parentheses
			par = re.sub(r'\[(.*?)\]', r'(\1)', par)
			# convert all inner nested parentheses to brackets
			n = 1
			while n >= 1:
				(par, n) = re.subn(r'\((.*?)\((\.*?)\)(.*?)\)', r'(\1[\2]\3)', par, flags=re.DOTALL)

			# process eventual multiparagraph scene
			if par.startswith('(') and par.count('(') > par.count(')'):
				# save eventual previous speech
				insert_speech(dpart_kind)

				text = '<p>%s</p>' % lxml.html.fromstring(par[1:]).text_content()
				within_scene = True
				continue
			if within_scene:
				if par.endswith(')') and par.count(')') > par.count('('):
					text += '\n\n<p>%s</p>' % lxml.html.fromstring(par[:-1]).text_content()
					insert_speech('scene')
					within_scene = False
				else:
					text += '\n\n<p>%s</p>' % lxml.html.fromstring(par).text_content()
				continue

			# process eventual new speaker
			# format `Doe, John, foreign minister`
			speech_start_pattern = r'<strong>(\w+), (\w+\.?)( (\w+\.?))?, (.*)</strong>'
			sp = re.match(speech_start_pattern, par, re.DOTALL)
			if sp:
				# save eventual previous speech
				insert_speech(dpart_kind)

				# identify speaker
				name = '%s %s' % (sp.group(2), sp.group(1))
				if (sp.group(4)):
					name = name.replace(' ', ' %s ' % sp.group(4))
				attribution = sp.group(5)
				if name in name_corrections:
					name = name_corrections[name]
				if len(name) == 0: continue
				speaker_id = mps.get(name)

				# create unknown speakers
				if not speaker_id:
					logging.warn('Speaker `%s, %s` not found, creating new Person' % (name, attribution))
					name_parts = re.match(r'(\w+\.?)( (\w+\.?))? (\w+)', name)
					person = {
						'name': name,
						'family_name': name_parts.group(4),
						'given_name': name_parts.group(1)
					}
					person['sort_name'] = '%s, %s' % (person['family_name'], person['given_name'])
					if name_parts.group(3):
						person['additional_name'] = name_parts.group(3)
						person['sort_name'] += ' %s' % person['additional_name']
					resp = vpapi.post('people', person)
					speaker_id = resp['id']
					mps[name] = speaker_id
				continue

			# remove HTML tags
			par = lxml.html.fromstring(par).text_content()

			# process eventual scene in this paragraph
			scene_pattern = r'(.*?)\(\s*([\d%s][^\(\)]{2,}[\.?!“])\s*\)(.*)$' % scrapeutils.CS_UPPERS
			while True:
				scene = re.match(scene_pattern, par, re.DOTALL)
				if not scene: break
				if scene.group(1):
					text += '\n\n<p>%s</p>' % scene.group(1).strip()
				insert_speech(dpart_kind)
				text = '<p>%s</p>' % scene.group(2).strip()
				insert_speech('scene')
				par = scene.group(3)

			if par:
				text += '\n\n<p>%s</p>' % par

		insert_speech(dpart_kind)

	if len(speeches) > 0:
		vpapi.post('speeches', speeches)
	logging.info('Scraped %s speeches' % len(speeches))
	speech_count += len(speeches)

	logging.info('Scraped %s speeches in total' % speech_count)
    def export_speeches(self):
        speeches = self.load_json('speeches')
        people = {}
        prefix_regex = re.compile(
            ur'(pred\u015bedavaju\u0107i )|(pred\u015bednik )|\
(generalni sekretar )', re.U)

        for p in vpapi.getall('people'):
            name = self.normalize_name(p['name'])
            people[name] = p['id']

        for speech in speeches:
            session_id = speech.get('event_id')
            speech['event_id'] = self.events_ids[session_id]
            url = speech['sources'][0]['url']
            if url.endswith('.pdf'):
                parsed_speeches = self.download_pdf(url)
                for n, s in enumerate(parsed_speeches):
                    text_speech = speech.copy()
                    text_speech['text'] = s['text']
                    text_speech['position'] = n + 1
                    text_speech['type'] = 'speech'

                    creator = self.normalize_name(s['creator'])
                    creator = prefix_regex.sub('', creator)

                    if creator in people:
                        text_speech['creator_id'] = people[creator]
                    else:
                        creator_id = None

                        for name in people:
                            if name in creator:
                                creator_id = people[name]
                                break

                        if creator_id is None:
                            resp = vpapi.getfirst(
                                'people', where={
                                    'name': {
                                        '$regex': s['creator'],
                                        'options': 'i'
                                    }
                                }
                            )
                            if resp is None:
                                self.log('Person "%(creator)s" not found. \
Creating one' % s, WARNING)
                                item = {
                                    'name': s['creator'],
                                    'sources': text_speech['sources']
                                }
                                resp = vpapi.post('people', item)
                            creator_id = resp['id']

                        people[creator] = creator_id
                        text_speech['creator_id'] = creator_id

                    self.get_or_create(
                        'speeches',
                        text_speech,
                        where_keys=['event_id', 'position']
                    )
            else:
                self.get_or_create('speeches', speech)
Beispiel #25
0
# set-up logging to a local file
if not os.path.exists(LOGS_DIR):
    os.makedirs(LOGS_DIR)
logname = datetime.utcnow().strftime('%Y-%m-%d-%H%M%S') + '.log'
logname = os.path.join(LOGS_DIR, logname)
logname = os.path.abspath(logname)
logging.basicConfig(level=logging.DEBUG,
                    format='%(message)s',
                    handlers=[logging.FileHandler(logname, 'w', 'utf-8')])
logging.getLogger('requests').setLevel(logging.ERROR)

logging.info(datetime.utcnow().strftime('%Y-%m-%d-%H:%M:%S') + '\tStarted')
db_log = vpapi.post('logs', {
    'status': 'running',
    'file': logname,
    'params': []
})

try:
    # get all senators by districts and all political groups
    baseurl = 'http://senat.cz/'
    people = []
    groups = {}
    iid = 100001

    ## people
    for i in range(1, 82):
        print(i)
        url = "http://senat.cz/senat/volby/hledani/o_obvodu.php?ke_dni=" + time.strftime(
            "%d") + "." + time.strftime("%m") + "." + time.strftime(
def savevotes(hl_poslanec):
    votes = {}
    voteevents = {}
    people = {}
    organizations = {}
    terms = {}
    for rowp in hl_poslanec:
        # if rowp[0] == 0: chybne hlasovani v db, viz http://www.psp.cz/sqw/hlasy.sqw?g=58297
        #  try:
        #    terms[hl_hlasovani[i][1].strip()]
        #  except:
        #    r_t = vpapi.get("organizations", where={'identifiers': {'$elemMatch': {"identifier": hl_hlasovani[0][1].strip(), "scheme": "psp.cz/organy"}}})
        #    for ident in r_t["_items"][0]["identifiers"]:
        #      if ident["scheme"] == "psp.cz/term":
        #        terms[hl_hlasovani[0][1].strip()] = ident["identifier"]
        try:
            voteevents[rowp[1].strip()]
        except:
            voteevents[rowp[1].strip()] = vpapi.get("vote-events", where={"identifier": rowp[1].strip()})
        r_voteevent = voteevents[rowp[1].strip()]

        try:
            people[rowp[0].strip()]
        except:
            people[rowp[0].strip()] = vpapi.get(
                "people",
                where={
                    "identifiers": {
                        "$elemMatch": {
                            "identifier": rowp[0].strip(),
                            "scheme": {"$regex": "psp.cz/poslanec/*", "$options": "i"},
                        }
                    }
                },
            )
        r_pers = people[rowp[0].strip()]

        try:
            organizations[r_pers["_items"][0]["id"]]
        except:
            organizations[r_pers["_items"][0]["id"]] = vpapi.get(
                "memberships", where={"person_id": r_pers["_items"][0]["id"]}, embed=["organization"]
            )
        r_org = organizations[r_pers["_items"][0]["id"]]

        for rowo in r_org["_items"]:
            if (
                rowo["organization"]["classification"] == "political group"
                and rowo["start_date"] <= r_voteevent["_items"][0]["start_date"]
            ):
                try:
                    rowo["end_date"]
                except:
                    fine = True
                else:
                    if rowo["end_date"] >= r_voteevent["_items"][0]["start_date"]:
                        fine = True
                    else:
                        fine = False
                    # 9 lines to overcome no python's function "isset" ... )-:
                if fine:
                    organization = rowo["organization"]
                    break
        vote = {
            "voter_id": r_pers["_items"][0]["id"],
            "option": option2option(rowp[2].strip()),
            "group_id": organization["id"],
            "vote_event_id": r_voteevent["_items"][0]["id"],
        }
        try:
            votes[r_voteevent["_items"][0]["id"]]
        except:
            votes[r_voteevent["_items"][0]["id"]] = []
        votes[r_voteevent["_items"][0]["id"]].append(vote.copy())
    #    for k in votes:
    #        vpapi.post("votes",votes[k])
    vpapi.post("votes", votes)
Beispiel #27
0
def scrape_motions(term):
	"""Scrape and save motions from the given term that are not scraped
	yet starting from the oldest ones. One Motion item, one VoteEvent
	item and many Vote items are created for each scraped motion detail
	page.

	Returns number of scraped motions.
	"""
	logging.info('Scraping motions of term `%s`' % term)

	# prepare mappings from source identifier to id for MPs and parliamentary groups
	chamber_id = get_chamber_id(term)

	people = vpapi.getall('people', projection={'identifiers': 1})
	mps = {mp['identifiers'][0]['identifier']: mp['id'] for mp in people if 'identifiers' in mp}

	orgs = vpapi.getall('organizations', where={'classification': 'parliamentary group', 'parent_id': chamber_id})
	parl_groups = {c['name']: c['id'] for c in orgs}

	# add differently spelled parliamentary groups
	group_corrections = {
		'2': {
			'Klub HZDS': 'Klub ĽS-HZDS',
			'Klub SMK': 'Klub SMK-MKP',
			'Klub Nezávislí': 'Klub Nezávislý',
		},
		'3': {
			'Klub HZDS': 'Klub ĽS-HZDS',
			'Klub SDKÚ': 'Klub SDKÚ-DS',
			'Klub Smer': 'Klub SMER-SD',
			'Klub Smer-SD': 'Klub SMER-SD',
			'Klub KNP': 'Klub nezávislých poslancov NR SR',
			'Klub Nezávislý': 'Klub nezávislých poslancov NR SR',
		},
	}
	for k, v in group_corrections.get(term, {}).items():
		parl_groups[k] = parl_groups[v]

	# prepare list of sessions that are not completely scraped yet
	sessions_to_scrape = []
	session_list = parse.session_list(term)
	for session in session_list['_items']:
		motions = parse.session(session['číslo'], term)
		if len(motions['_items']) == 0: continue
		last_motion_id = motions['_items'][-1]['id']
		m_url = 'http://www.nrsr.sk/web/Default.aspx?sid=schodze/hlasovanie/hlasklub&ID=%s' % last_motion_id
		existing = vpapi.getfirst('motions', where={'sources.url': m_url})
		if existing: break
		sessions_to_scrape.append((session, motions))

	# scrape motions from those sessions
	scraped_motions_count = 0
	for s, motions in reversed(sessions_to_scrape):
		logging.info('Scraping session `%s`' % s['názov'])

		# insert the session event unless it already exists
		session = {
			'name': s['názov'],
			'identifier': s['číslo'],
			'organization_id': chamber_id,
			'type': 'session',
		}
		try:
			session['start_date'] = sk_to_utc(s['trvanie']) + 'T00:00:00'
			session['end_date'] = session['start_date']
		except ValueError:
			# multiday session contains votes; dates are set by debates scraping
			pass
		key = ('organization_id', 'type', 'identifier')
		session_id, _ = get_or_create('events', session, key)

		for i, m in enumerate(motions['_items']):
			# check if the motion is already present
			m_id = re.search(r'ID=(\d+)', m['url']['výsledok']).group(1)
			# we not use directly m['url']['kluby'] because it is not always present
			m_url = 'http://www.nrsr.sk/web/Default.aspx?sid=schodze/hlasovanie/hlasklub&ID=%s' % m_id
			existing = vpapi.getfirst('motions', where={'sources.url': m_url})
			if existing: continue

			try:
				motion_id = None
				vote_event_id = None

				# insert motion
				logging.info('Scraping motion %s of %s (voted at %s)' % (i+1, len(motions['_items']), m['dátum']))
				parsed_motion = parse.motion(m['id'])
				motion = {
					'organization_id': chamber_id,
					'legislative_session_id': session_id,
					'identifier': parsed_motion['číslo'],
					'text': parsed_motion['názov'],
					'date': sk_to_utc(m['dátum']),
					'sources': [{
						'url': parsed_motion['url'],
						'note': 'Hlasovanie na webe NRSR'
					}],
				}
				if 'výsledok' in parsed_motion:
					motion['result'] = 'pass' if parsed_motion['výsledok'] == 'Návrh prešiel' else 'fail'
				resp = vpapi.post('motions', motion)
				motion_id = resp['id']

				# insert vote event
				vote_event = {
					'motion_id': motion_id,
					'organization_id': chamber_id,
					'legislative_session_id': session_id,
					'identifier': parsed_motion['číslo'],
					'start_date': motion['date'],
					'sources': [{
						'url': parsed_motion['url'],
						'note': 'Hlasovanie na webe NRSR'
					}],
				}
				if 'výsledok' in parsed_motion:
					vote_event['result'] = motion['result']
				if 'súčty' in parsed_motion:
					options = {
						'yes': '[z] za',
						'no': '[p] proti',
						'abstain': '[?] zdržalo sa',
						'absent': '[0] neprítomní',
						'not voting': '[n] nehlasovalo'
					}
					vote_event['counts'] = [
						{'option': o, 'value': int(parsed_motion['súčty'][s])}
						for o, s in options.items() if parsed_motion['súčty'][s] != ''
					]
					if len(vote_event['counts']) == 0:
						del vote_event['counts']
				resp = vpapi.post('vote-events', vote_event)
				vote_event_id = resp['id']

				# insert votes
				if 'hlasy' in parsed_motion and len(parsed_motion['hlasy']) > 0:
					vote_options = {
						'z': 'yes',
						'p': 'no',
						'?': 'abstain',
						'n': 'not voting',
						'0': 'absent'
					}
					votes = []
					for v in parsed_motion['hlasy']:
						# skip MPs not applying their mandate
						if v['hlas'] == '-': continue
						pg = normalize_parlgroup_name(v['klub'])
						votes.append({
							'vote_event_id': vote_event_id,
							'option': vote_options[v['hlas']],
							'voter_id': mps.get(v['id']),
							'group_id': parl_groups.get(pg),
						})
					if len(votes) > 0:
						resp = vpapi.post('votes', votes)

			# delete incomplete data if insertion of the motion, vote event or votes failed
			except:
				if motion_id:
					vpapi.delete('motions', motion_id)
				if vote_event_id:
					vpapi.delete('vote-events', vote_event_id)
				raise

			scraped_motions_count += 1

	logging.info('Scraped %s motions of term `%s`' % (scraped_motions_count, term))
	return scraped_motions_count
                                rowo["end_date"]
                            except:
                                fine = True
                            else:
                                if rowo["end_date"] >= r_voteevent["_items"][0]["start_date"]:
                                    fine = True
                                else:
                                    fine = False
                                # 9 lines to overcome no python's function "isset" ... )-:
                            if fine:
                                organization = rowo["organization"]
                                break
                    vote = {
                        "voter_id": r_pers["_items"][0]["id"],
                        "option": option2option(rowp[2].strip()),
                        "group_id": organization["id"],
                        "vote_event_id": r_voteevent["_items"][0]["id"],
                    }
                    try:
                        votes[r_voteevent["_items"][0]["id"]]
                    except:
                        votes[r_voteevent["_items"][0]["id"]] = []
                    votes[r_voteevent["_items"][0]["id"]].append(vote.copy())
            for k in votes:
                vpapi.post("votes", votes[k])
                print(j)
                j = j + 1

        except:
            nothing = 1
Beispiel #29
0
def scrape_old_debates(term):
	"""Scrape and save speeches from debates of the given term, one
	of those older terms where transcripts of debates are stored in
	RTF files.

	Returns number of scraped speeches.
	"""

	def insert_speech(type):
		"""Insert a speech entity with the given type and data
		from parent scope variables and update end date of the
		corresponding session and sitting. Delete `text`
		variable."""
		nonlocal text, position
		if not text: return
		position = position + 1
		speech = {
			'text': text.strip().replace('[', '(').replace(']', ')'),
			'type': type,
			'position': position,
			'event_id': sitting_id,
			'sources' : [{
				'url': debate['url'],
				'note': 'Prepis debaty v Digitálnej knižnici na webe NRSR'
			}]
		}
		if type != 'scene':
			speech['creator_id'] = speaker_id
			speech['attribution_text'] = attribution.strip()
		speeches.append(speech)
		text = ''

		if date > session_end_date:
			vpapi.patch('events', session_id, {'end_date': date})
		if date > sitting_end_date:
			vpapi.patch('events', sitting_id, {'end_date': date})

	logging.info('Scraping debates of term `%s`' % term)
	chamber_id = get_chamber_id(term)

	# prepare mapping from MP's name to id
	people = vpapi.getall('people', projection={'given_name': 1, 'additional_name': 1, 'family_name': 1})
	mps = {}
	for mp in people:
		if 'additional_name' in mp:
			name = '%s. %s. %s' % (mp['given_name'][0], mp['additional_name'][0], mp['family_name'])
		else:
			name = '%s. %s' % (mp['given_name'][0], mp['family_name'])
		mps[name] = mp['id']

	# load name corrections
	with open(os.path.join(CONF_DIR, 'name_corrections.json'), encoding='utf8') as f:
		name_corrections = json.load(f)

	# scrape list of debates
	debates = parse.old_debates_list(term)

	# add the debate missing in the list
	if term == '4':
		debates['_items'].append({
			'názov': 'Autorizovaná rozprava, 48. schôdza NR SR, 3. 2. 2010',
			'id': '2010_02_03',
			'url': 'http://www.nrsr.sk/dl/Browser/DsDocument?documentId=391413'
		})

	speech_count = 0
	session_identifier = None
	for debate in debates['_items']:
		# skip obsolete debates in the list
		if term == '1':
			if (debate['názov'] == 'Stenozáznam' and debate['id'] != '198550' or
					debate['id'] in ('65890', '65945', '65949')):
				continue
		elif term == '2':
			if debate['názov'].startswith('Stenografická') and debate['id'] != '92098':
				continue
		elif term == '3':
			if debate['id'] == '181047':
				continue

		logging.info('Scraping debate `%s` (id=%s)' % (debate['názov'], debate['id']))
		if term == '1':
			paragraphs = parse.debate_of_term1(debate['id'])
		else:
			paragraphs = parse.debate_of_terms234(debate['id'])

		# normalize header of the debate transcript
		if term == '2':
			# join first 4 paragraphs and add trailing underscores to mark the header
			paragraphs = ['%s %s %s %s\n___' % (paragraphs[0], paragraphs[1], paragraphs[2],
				paragraphs[3])] + paragraphs[4:]
		elif term in ('3', '4'):
			# join first paragraphs until " hodine" ending is found
			# and add trailing underscores to mark the header
			p = ''
			while True:
				p += ' ' + paragraphs.pop(0)
				if p.endswith('hodine'): break
			if paragraphs[0].startswith('___'):
				paragraphs.pop(0)
			paragraphs.insert(0, p + '\n___')

		# extract speeches from the debate
		speeches = []
		text = ''
		within_scene = False
		for par in paragraphs:
			par = par.replace('\n', ' ').strip()
			if not par: continue

			# fix last scene
			if re.search(r'\b(skončil.|skončené|prerušené|Prerušenie rokovani[ae])\s+o\s+(.*?)\s+hodine.', par):
				if not par[0] in ('(', '[', '/'):
					par = '(%s)' % par

			# convert brackets to parentheses
			par = re.sub(r'\[(.*?)\]', r'(\1)', par)
			# slash pairs are converted to parentheses too in term 1
			if term == '1':
				par = re.sub(r'(^|\s)/(.*?)/(\s|$)', r'\1(\2)\3', par)
			# convert all inner nested parentheses to brackets
			n = 1
			while n >= 1:
				(par, n) = re.subn(r'\((.*?)\((.*?)\)(.*?)\)', r'(\1[\2]\3)', par, flags=re.DOTALL)

			# process eventual multiparagraph scene
			if par.startswith('(') and par.count('(') > par.count(')'):
				# save eventual previous speech
				insert_speech('speech')

				text = '<p>%s</p>' % par[1:]
				within_scene = True
				continue
			if within_scene:
				if par.endswith(')') and par.count(')') > par.count('('):
					text += '\n\n<p>%s</p>' % par[:-1]
					insert_speech('scene')
					within_scene = False
				else:
					text += '\n\n<p>%s</p>' % par
				continue

			# process eventual header
			header_pattern = r'((\(?(\d+)\.\)?\s+schôdz)|slávnostn).*?(\d+)\..*\b(\w{3,})\s+(\d{4})(.*?)_{3,}$'
			hd = re.search(header_pattern, par, re.DOTALL)
			if hd:
				# save eventual previous speech
				insert_speech('speech')

				sk_date = '%s. %s %s' % (hd.group(4), hd.group(5), hd.group(6))
				initial_time = re.search(r'\s+o\s+(.*?)\s+hodine', hd.group(7), re.DOTALL)
				if initial_time and initial_time.group(1) != '??':
					h, m = initial_time.group(1).strip('.').split('.')
					date = sk_to_utc(sk_date + ' %s:%s:00' % (h.strip().zfill(2), m.strip().zfill(2)))
				else:
					date = sk_to_utc(sk_date) + 'T00:00:00'

				if hd.group(1).startswith('sláv'):
					new_session_name = 'Mimoriadna schôdza'
					if term == '1':
						new_session_identifier = debate['časť']
					elif term == '2':
						new_session_identifier = '1000'
					else:
						sl = parse.session_list(term)
						d = '%s. %s. %s' % (int(date[8:10]), int(date[5:7]), int(date[0:4]))
						new_session_identifier = next((s['číslo'] for s in sl['_items'] if s['trvanie'] == d))
				else:
					new_session_name = '%s. schôdza' % hd.group(3)
					new_session_identifier = hd.group(3)

				if new_session_identifier != session_identifier:
					# create new session event
					session = {
						'name': new_session_name,
						'identifier': new_session_identifier,
						'organization_id': chamber_id,
						'type': 'session',
						'start_date': date,
					}
					key = ('organization_id', 'type', 'identifier')
					session_id, _ = get_or_create('events', session, key)
					session_identifier = new_session_identifier
					session_end_date = date
					sitting_count = 0

				# create new sitting event
				sitting_count += 1
				sitting = {
					'name': '%s. deň rokovania, %s' % (sitting_count, sk_date),
					'identifier': str(sitting_count),
					'organization_id': chamber_id,
					'type': 'sitting',
					'start_date': date,
					'parent_id': session_id,
				}
				key = ('parent_id', 'type', 'identifier')
				sitting_id, created = get_or_create('events', sitting, key)
				sitting_end_date = date
				position = 0

				# delete existing speeches of the sitting
				if not created:
					obsolete = vpapi.getall('speeches', where={'event_id': sitting_id})
					for speech in obsolete:
						vpapi.delete('speeches', speech['id'])
				continue

			# process eventual start of a speech
			if date < '2001-09-04':
				# format `Foreign minister J. Doe:`
				speech_start_pattern = r'(.*?)\b([^\W\d])\.[\s_]+((\w)\.[\s_]+)?([\w-]+):$'
			else:
				# format `J. Doe, foreign minister: speech`
				speech_start_pattern = r'([^\W\d])\.[\s_]+((\w)\.[\s_]+)?([\w-]+),\s+(.+?):(.+)$'
			sp = re.match(speech_start_pattern, par, re.DOTALL)
			if sp:
				# save eventual previous speech
				insert_speech('speech')

				# identify speaker
				if date < '2001-09-04':
					name = '%s. %s' % (sp.group(2), sp.group(5))
					if (sp.group(4)):
						name = name.replace(' ', ' %s. ' % sp.group(4))
					attribution = sp.group(1)
					par = ''
				else:
					name = '%s. %s' % (sp.group(1), sp.group(4))
					if (sp.group(3)):
						name = name.replace(' ', ' %s. ' % sp.group(3))
					attribution = sp.group(5)
					par = sp.group(6)

				if name in name_corrections:
					name = name_corrections[name]
				attribution = attribution[0].lower() + attribution[1:].strip()
				speaker_id = mps.get(name)

				# create unknown speakers
				if not speaker_id:
					logging.warn('Speaker `%s, %s` not found, creating new Person' % (name, attribution))
					name_parts = re.match(r'(\w)\. ((\w)\. )?(\w+)', name)
					person = {
						'name': name,
						'family_name': name_parts.group(4),
						'given_name': name_parts.group(1)
					}
					person['sort_name'] = '%s, %s.' % (person['family_name'], person['given_name'])
					if name_parts.group(3):
						person['additional_name'] = name_parts.group(3)
						person['sort_name'] += ' %s.' % person['additional_name']
					resp = vpapi.post('people', person)
					speaker_id = resp['id']
					mps[name] = speaker_id

			# recognize date(-time) stamps in transcripts
			ds = re.match(r'^\s*(\d+\.\s\w+\s\d{4})(.*hodine)?\s*$', par)
			if ds:
				dt = ds.group(1).strip()
				tm = re.search(r'o\s+(.*?)\s+', ds.group(2) or '')
				try:
					if tm:
						h, m = tm.group(1).strip('.').split('.')
						date = sk_to_utc('%s %s:%s:00' % (dt, h.strip().zfill(2), m.strip().zfill(2)))
					else:
						date = sk_to_utc(dt) + 'T00:00:00'
					continue
				except ValueError:
					pass

			# process eventual scene in this paragraph
			scene_pattern = r'(.*?)\(\s*([\d%s][^\(\)]{2,}[\.?!“])\s*\)(.*)$' % scrapeutils.CS_UPPERS
			while True:
				scene = re.match(scene_pattern, par, re.DOTALL)
				if not scene: break
				if scene.group(1):
					text += '\n\n<p>%s</p>' % scene.group(1).strip()
				insert_speech('speech')
				text = '<p>%s</p>' % scene.group(2).strip()
				insert_speech('scene')
				par = scene.group(3)

			if par:
				text += '\n\n<p>%s</p>' % par.strip()

		insert_speech('speech')

		# extract end time of the session
		final_time = re.search(
			r'\b(skončil.|skončené|prerušené|Prerušenie rokovani[ae])\s+o\s+(.*?)\s+hodine.',
			speeches[-1]['text'])
		if final_time:
			tm = final_time.group(2)
			tm = tm.replace('O', '0').replace(',', '.')
			h, m = tm.strip('.').split('.')
			final_date = '%s.%s.%s %s:%s:00' % (date[8:10], date[5:7], date[0:4], h.strip().zfill(2), m.strip().zfill(2))
			final_date = sk_to_utc(final_date)
			vpapi.patch('events', session_id, {'end_date': final_date})
			vpapi.patch('events', sitting_id, {'end_date': final_date})

		vpapi.post('speeches', speeches)
		logging.info('Scraped %s speeches' % len(speeches))
		speech_count += len(speeches)

	logging.info('Scraped %s speeches in total' % speech_count)
Beispiel #30
0
                    try:
                        votes[r_voteevent["_items"][0]["id"]]
                    except:
                        votes[r_voteevent["_items"][0]["id"]] = []
                    votes[r_voteevent["_items"][0]["id"]].append(vote.copy())
                    j = j + 1
                    print(str(j) + ':' + str(j/200))


            j = 0
            votesli = []
            n = 0
#            raise(Exception)
            for k in votes:
                if (j == 1):
                    vpapi.post("votes",votesli)
                    votesli = []
                    print(str(n) + "/" + str(len(votes)))
                    print(k)
                    j = 0
                j = j + 1
                n += 1
                votesli = votesli + votes[k]
#            vpapi.post("votes",votesli)
#                vpapi.post("votes",votes[k])
#            for k in votes:
#                votesli = votesli + votes[k]
#            vpapi.post("votes",votesli)
        except:
            nothing = 1
    #         logging.warning('Something went wrong with year ' + str(term) + 'and file ' + str(i) + ' (it may not exist), last vote_event_id: ' + str(last_ve_id))
 def batch_create(self, endpoint, items):
     resp = vpapi.post(endpoint, items)
     if resp['_status'] != 'OK':
         raise Exception(resp)
     self.log('Created %d items' % len(resp['_items']), DEBUG)
     return
Beispiel #32
0
# set-up logging to a local file
if not os.path.exists(LOGS_DIR):
    os.makedirs(LOGS_DIR)
logname = datetime.utcnow().strftime('%Y-%m-%d-%H%M%S') + '.log'
logname = os.path.join(LOGS_DIR, logname)
logname = os.path.abspath(logname)
logging.basicConfig(level=logging.DEBUG,
                    format='%(message)s',
                    handlers=[logging.FileHandler(logname, 'w', 'utf-8')])
logging.getLogger('requests').setLevel(logging.ERROR)

logging.info(datetime.utcnow().strftime('%Y-%m-%d-%H:%M:%S') + '\tStarted 2')
db_log = vpapi.post('logs', {
    'status': 'running',
    'file': logname,
    'params': []
})

vpapi.parliament('cz/senat')
vpapi.authorize(authentication.username, authentication.password)
vpapi.timezone('Europe/Prague')

o2id = {}
organizations = vpapi.getall("organizations")
for org in organizations:
    o2id[org['name']] = org['id']

p2id = {}
persons = vpapi.getall('people')
for p in persons:
Beispiel #33
0
def scrape(countries, people, votes):
    global effective_date
    effective_date = date.today().isoformat()

    # execute MP's bio data.
    georgia = georgia_scraper.GeorgiaScraper()
    armenia = armenia_scraper.ArmeniaScraper()
    ukraine = ukraine_scraper.UkraineScraper()
    belarus_lowerhouse = belarus_lowerhouse_scraper.BelarusLowerhouseScraper()
    belarus_upperhouse = belarus_upperhouse_scraper.BelarusUpperhouseScraper()
    moldova = moldova_scraper.MoldovaScraper()
    references = {"georgia": georgia, "armenia": armenia, "ukraine": ukraine,
                  "belarus-lowerhouse": belarus_lowerhouse, "moldova": moldova,
                  "belarus-upperhouse": belarus_upperhouse}
    countries_array = []
    if countries == "all":
        for key in references:
            countries_array.append(key)
    else:
        countries_array = countries.split(',')
        indexes = []
        for country in countries_array:
            if country.lower() not in references:
                indexes.append(countries_array.index(country))
        if len(indexes) > 0:
            countries_array.pop(indexes)
    with open(os.path.join(BASE_DIR, 'access.json')) as f:
        creds = json.load(f)
    if len(countries_array) > 0:
        for item in sorted(countries_array):
            if internet_on(): # scrape and post data from parliaments if there's internet connection
                print "\n\tPosting and updating data from %s parliament" % item
                print "\tThis may take a few minutes..."
                vpapi.parliament(creds[item.lower()]['parliament'])
                vpapi.timezone(creds[item.lower()]['timezone'])
                vpapi.authorize(creds[item.lower()]['api_user'], creds[item.lower()]['password'])
                if people == "yes":
                    members = references[item.lower()].scrape_mp_bio_data()
                    chamber = references[item.lower()].scrape_chamber()
                    parliamentary_groups = references[item.lower()].scrape_parliamentary_groups()
                    committee = references[item.lower()].scrape_committee()
                    data_collections = {
                        "a-people": members,
                        "b-chamber": chamber,
                        "c-parliamentary_groups": parliamentary_groups,
                        "d-committe": committee
                    }
                    # inserts data for each data collection in Visegrad+ Api
                    for collection in sorted(set(data_collections)):
                        widgets = ['        Progress: ', Percentage(), ' ', Bar(marker='#', left='[', right=']'),
                                   ' ', ETA(), " - Processed: ", Counter(), ' items             ']
                        pbar = ProgressBar(widgets=widgets)
                        print "\n\tPosting and updating data to the Visegrad+ from %s data collection\n\n" % \
                              collection[2:]
                        if len(data_collections[collection]) > 0:
                            for json_doc in pbar(data_collections[collection]):
                                if collection == "a-people":
                                    where_condition = {'identifiers': {'$elemMatch': json_doc['identifiers'][0]}}
                                    collection_of_data = "people"
                                elif collection == "c-parliamentary_groups" or collection == "d-committe":
                                    if item.lower() == "armenia" or item.lower() == "belarus-upperhouse"\
                                            or item.lower() == "ukraine":
                                        where_condition = {'name': json_doc['name'], "parent_id": json_doc['parent_id']}
                                    else:
                                        where_condition = {'name': json_doc['name']}
                                    collection_of_data = "organizations"
                                elif collection == "b-chamber":
                                    where_condition = {'identifiers': {'$elemMatch': json_doc['identifiers'][0]}}
                                    collection_of_data = "organizations"

                                existing = vpapi.getfirst(collection_of_data, where=where_condition)
                                if not existing:
                                    resp = vpapi.post(collection_of_data, json_doc)
                                else:
                                    json_obj_id = existing['id']
                                    items_to_delete = ["created_at", "updated_at", "_links", "id"]
                                    for item_delete in items_to_delete:
                                        del existing[item_delete]
                                    if json.loads(json.dumps(json_doc)) == existing:
                                        continue
                                    else:
                                        resp = vpapi.put(collection_of_data, json_obj_id, json_doc, effective_date=effective_date)

                                    # update by PUT is preferred over PATCH to correctly remove properties that no longer exist now
                                if resp["_status"] != "OK":
                                    raise Exception("Invalid status code")

                            print "\n\tFinished Posting and updating data from %s data collection\n" % collection[2:]
                    if item.lower() != "georgia":
                        memberships = {
                            "chambers": references[item.lower()].scrape_membership(),
                            "parliamentary_groups": references[item.lower()].scrape_parliamentary_group_membership(),
                            "committees": references[item.lower()].scrape_committee_members()
                        }
                    elif item.lower() == "georgia":
                        memberships = {
                            "chambers": references[item.lower()].scrape_membership()
                        }

                    for data_collection in memberships:
                        widgets_stat = ['        Progress: ', Percentage(), ' ', Bar(marker='#', left='[', right=']'),
                                        ' ', ETA(), " - Processed: ", Counter(), ' items             ']
                        prog_bar = ProgressBar(widgets=widgets_stat)
                        if len(memberships[data_collection]) > 0:
                            print "\n\tPosting and updating data from %s membership data collection\n" % data_collection
                            for json_doc in prog_bar(memberships[data_collection]):
                                existing = vpapi.getfirst("memberships", where={'organization_id': json_doc['organization_id'],
                                                                                "person_id": json_doc['person_id']})
                                if not existing:
                                    resp = vpapi.post("memberships", json_doc)
                                else:
                                    json_obj_id = existing['id']
                                    items_to_delete = ["created_at", "updated_at", "_links", "id"]
                                    for item_delete in items_to_delete:
                                        del existing[item_delete]
                                    if json.loads(json.dumps(json_doc)) == existing:
                                        continue
                                    else:
                                        resp = vpapi.put("memberships", json_obj_id, json_doc, effective_date=effective_date)
                                if resp["_status"] != "OK":
                                    raise Exception("Invalid status code")
                            print "\n\tFinished Posted and updated data from %s membership data collection\n" % data_collection
                        else:
                            print "\n\tThere is no data from %s membership data collection\n" % data_collection
                            continue
                if votes == "yes":
                    if item.lower() == "ukraine":
                        events = references[item.lower()].scrape_events()
                        try:
                            if len(events) > 0:
                                widgets_events = ['        Progress: ', Percentage(), ' ', Bar(marker='#', left='[', right=']'),
                                           ' ', ETA(), " - Processed: ", Counter(), ' items             ']
                                pbar_events = ProgressBar(widgets=widgets_events)
                                for json_doc in pbar_events(events):
                                    existing_event = vpapi.getfirst("events", where={'identifier': json_doc['identifier']})
                                    if not existing_event:
                                        resp = vpapi.post("events", json_doc)
                                    else:
                                        resp = vpapi.put("events", json_doc['id'], json_doc, effective_date=effective_date)
                                    if resp["_status"] != "OK":
                                        raise Exception("Invalid status code")
                                print "\n\tFinished Posting and updating data from events data collection"
                            else:
                                print "\n\tThere are no new events"
                        except BaseException as ex:
                            print ex.message
                        else:
                            print "\tThere's not any event to post from %s parliament" % item
                        motions_vote_events = references[item.lower()].vote_events()
                        voting_results = references[item.lower()].scrape_votes()
                        try:
                            if len(voting_results) > 0:
                                resp = vpapi.post("votes", voting_results)
                                if resp["_status"] != "OK":
                                    raise Exception("Invalid status code")
                                print "\n\tFinished Posting and updating data from votes data collection"
                        except BaseException as ex:
                            print ex.message
                    elif item.lower() == "georgia":
                        voting_data_collections = {
                            "amotions": references[item.lower()].motions(),
                            "bvote-events": references[item.lower()].vote_events(),
                        }
                        votes = references[item.lower()].scrape_votes()
                        for collection in sorted(voting_data_collections):
                            try:
                                if len(voting_data_collections[collection]) > 0:
                                    resp = vpapi.post(collection[1:], voting_data_collections[collection])
                                    if resp["_status"] != "OK":
                                        raise Exception("Invalid status code")
                                    print "\n\tFinished Posting and updating data from %s data collection" % collection[1:]
                            except BaseException as ex:
                                print ex.message

                        print "\n\tPosting voting records from Georgia Parliament\n"
                        try:
                            if len(votes) > 0:
                                vpapi.post("votes", votes)
                            print "\n\tFinished Posting and updating data from votes data collection"
                        except BaseException as ex:
                            print ex.message
                    else:
                        print "\n\tThere are no voting records for %s" % item
                vpapi.deauthorize()
            else:
                print "\n\tInternet connection problems for %s official parliament web page" % item
                continue
    else:
        print "\n\tInvalid country/ies added"
Beispiel #34
0
        motion['requirement'] = guess_majority(quorum, present)

        h2s = domtree1.xpath('//h2')
        h2s.pop(0)
        j = 0
        tables.pop(0)
        for table in tables:
            tds = table.xpath('tr/td')
            for td in tds:
                li = td.text.strip().split('\xa0')
                vote = {
                    "vote_event_id":
                    iid,
                    "voter_id":
                    pp2id(" ".join([li[2], li[3]]), vote_event['start_date'],
                          p2id),
                    "option":
                    option2option(li[0]),
                    "group_id":
                    o2id[h2s[j].text.strip()]
                }
                votes.append(vote)
            j += 1

        vpapi.post("motions", motion)
        vpapi.post("vote-events", vote_event)
        vpapi.post("votes", votes)
    except:
        print("XXX:" + iid)
        nothing = 0  # "Zmatečné hlasování"