示例#1
0
    def get_urls(self):
        url = DIPUTADOS_URL
        has_next = True
        url_fichas = []

        while(has_next):
            time.sleep(1)
            print url
            try:
                html_doc = get_url(url)
                soup = BeautifulSoup(html_doc)
                list_dip = soup.findAll(lambda tag: tag.name == 'a' and tag.parent.name == 'li' and tag.has_key('class'), "")
                for dip in list_dip:
                    url_fichas.append(dip.attrs['href'].__str__())

                pagination = soup.find(lambda tag: tag.name == 'a' and tag.parent.name == 'ul' and tag.has_key('class'), "", text='Página Siguiente')
                if not pagination:
                    has_next = False
                else:
                    url = pagination.attrs['href'].__str__()
            except:
                has_next = False
                #TODO send error mail

        return url_fichas
 def get_session(self, url):
     html_doc = get_url(url)
     if html_doc:
         soup = BeautifulSoup(html_doc)
         xml_soup = soup.find(lambda tag: tag.name == 'a' and tag.parent.name == 'td')
         if xml_soup:
             print xml_soup
             xml_url = 'http://www.congreso.es' + xml_soup.attrs['href']
             self.common_handle(xml_url)
示例#3
0
 def get_session(self, url):
     html_doc = get_url(url)
     if html_doc:
         soup = BeautifulSoup(html_doc)
         xml_soup = soup.find(
             lambda tag: tag.name == 'a' and tag.parent.name == 'td')
         if xml_soup:
             print xml_soup
             xml_url = 'http://www.congreso.es' + xml_soup.attrs['href']
             self.common_handle(xml_url)
示例#4
0
    def parser_dip(self, url_fichas):
        # get result
        for url in url_fichas:
            print url
            html_doc = get_url(url)
            soup = BeautifulSoup(html_doc)

            cv_soup = soup.find(id='curriculum')
            if cv_soup:
                query = urlparse.parse_qs(urlparse.urlparse(url).query)
                congress_id = query['idDiputado'][0]
                datos_diputado = soup.find(id='datos_diputado')
                member_instance, member_created = Member.objects.get_or_create(congress_id=congress_id)
                member_instance.congress_web = url

                #get name
                soup_name = cv_soup.find('div', 'nombre_dip')
                if soup_name:
                    member_instance.name = soup_name.text.split(',')[1].strip().encode('utf8')
                    member_instance.second_name = soup_name.text.split(',')[0].strip().encode('utf8')

                #get url
                member_instance.ficha = url

                #get email
                soup_email = cv_soup.find(lambda tag: tag.name == 'a' and tag.parent.name == 'div' and tag.has_key('title'), text=re.compile("([a-zA-Z0-9]*[\.])*@congreso.es"))
                if soup_email:
                    member_instance.email = soup_email.text.split()[0].encode('utf8')

                #get web
                soup_web = cv_soup.find(lambda tag: tag.name == 'a' and tag.parent.name == 'div' and tag.has_key('title'), text=re.compile("https?:\/\/"))
                if soup_web:
                    member_instance.web = soup_web.attrs['href'].encode('utf8')

                #get twitter
                soup_twitter = cv_soup.findAll(lambda tag: tag.name == 'img' and tag.parent.name == 'a',src=re.compile("codigoTipoDireccion=tw"))
                if soup_twitter:
                    member_instance.twitter = soup_twitter[0].parent.attrs['href'].encode('utf8')

                #get division
                soup_div = cv_soup.find('div', 'dip_rojo', text=re.compile("Diputad[ao] por [a-zA-Z]?"))
                if soup_div:
                    soup_div = re.sub(r'Diputad[ao] por ', '' ,soup_div.text).strip()
                    soup_div = re.sub(r'\.$', '' ,soup_div)
                    member_instance.division = soup_div.encode('utf8')

                #get foto
                if datos_diputado:
                    avatar_soup = datos_diputado.find(lambda tag: tag.name == 'img' and tag.parent.name == 'p')
                    if avatar_soup:
                        member_instance.avatar = 'http://www.congreso.es' + avatar_soup.attrs['src']

                member_instance.save()

                #get grupo
                group_soup = cv_soup.find(lambda tag: tag.name == 'a' and tag.parent.name == 'div', text=re.compile("G\.P\. .* \( [a-zA-Z]+ \)"))
                if group_soup:
                    scrap_group(group_soup.attrs['href'], ACTUAL_TERM)
                    group_instance = Group.objects.get(congress_url=group_soup.attrs['href'])

                    #get party
                    party_soup = soup.find('p', 'nombre_grupo')
                    if party_soup:
                        party_name = party_soup.text.strip().encode('utf8')
                        party_instance, party_created = Party.objects.get_or_create(name=party_name)
                        party_instance.name = party_name
                        if datos_diputado:
                            logo_party = datos_diputado.find(lambda tag: tag.name == 'img' and tag.parent.name == 'a')
                            if logo_party:
                                logo_url = 'http://www.congreso.es' + logo_party.attrs['src']
                                logo_name = logo_url.split('/')[-1].encode('utf8')
                                if not party_instance.logo:
                                    save_url_image(party_instance.logo, logo_url.encode('utf8'), logo_name)
                                party_instance.web = logo_party.parent.attrs['href'].encode('utf8')
                        party_instance.group = group_instance
                        #TODO partygroup instance
                        groupparty_instance, groupparty_created = GroupParty.objects.get_or_create(party=party_instance, group=group_instance)
                        groupparty_instance.save()

                        party_instance.save()

                        memberparty_instance, memberparty_created = MemberParty.objects.get_or_create(party=party_instance, member=member_instance)
                        memberparty_instance.save()


                """