def getcapitalresults(url): raw_html = webfunctions.simple_get(url) html = BeautifulSoup(raw_html, 'html.parser') if re.search( 'Főpolgármester-választás egyéni választókerületi szavazás eredményeinek megjelenítése', html.findAll('div', {'class': 'nvi-electoral-district-filter'})[0].text): table = html.find(lambda tag: tag.name == 'table') if not table: return None else: rows = table.findAll(lambda tag: tag.name == 'tr') l1 = [] l2 = [] l3 = [] for tr in rows: columns = tr.find_all('span') l1.append(columns[0].text.strip()) l3.append( int( re.search('(.*)\(', columns[2].text.strip()).group(0)[0:-2])) parties = ';'.join([ span.text.strip() for span in tr.find_all( 'span', { 'class': '_onkszavazokorieredmenyek_WAR_nvinvrportlet_popover' }) ]) l2.append(parties) df_capital = pd.DataFrame([l1, l2, l3]) return df_capital else: return None
def getdistrict(maz, taz, idx): maz = padtolen(maz, 2) taz = padtolen(taz, 3) url = 'https://www.valasztas.hu/szavazokorok_onk2019?p_p_id=onkszavazokorieredmenyek_WAR_nvinvrportlet&p_p_lifecycle=1&p_p_state=maximized&p_p_mode=view&_onkszavazokorieredmenyek_WAR_nvinvrportlet_tabId=tab2&_onkszavazokorieredmenyek_WAR_nvinvrportlet_telepulesKod=' + taz + '&_onkszavazokorieredmenyek_WAR_nvinvrportlet_megyeKod=' + maz + '&_onkszavazokorieredmenyek_WAR_nvinvrportlet_vlId=294&_onkszavazokorieredmenyek_WAR_nvinvrportlet_vltId=687&_onkszavazokorieredmenyek_WAR_nvinvrportlet_szavkorSorszam=' + str( idx) raw_html = webfunctions.simple_get(url) html = BeautifulSoup(raw_html, 'html.parser') table = html.findAll('div', {'class': 'szavazokorieredmenyek-details-container'}) if table: district = [] attrs = {} district_table = table[0].findAll('span', {'class': 'text-semibold'}) attrs['href'] = url attrs['maz'] = int(maz) attrs['taz'] = int(taz) attrs['sorsz'] = int(idx) attrs['cimk'] = district_table[0].text.strip() attrs['cimt'] = re.search( '^\D+', district_table[1].text.strip()).group(0).strip() if district_table[2].text.strip() == '-': attrs['evk'] = 0 else: attrs['evk'] = int( re.search('^\D+(.*)\.', district_table[2].text.strip()).group(1)) attrs['tip'] = 0 district.append(attrs) return district else: return None
def getcountyresults(url): raw_html = webfunctions.simple_get(url) html = BeautifulSoup(raw_html, 'html.parser') if re.search( 'Megyei közgyűlés választása egyéni választókerületi szavazás eredményeinek megjelenítése', html.findAll('div', {'class': 'nvi-electoral-district-filter'})[0].text): table = html.find(lambda tag: tag.name == 'table') l1 = [] l2 = [] if not table: return None else: rows = table.findAll(lambda tag: tag.name == 'tr') for tr in rows: columns = tr.find_all('span', {'class': 'nvi-text-normal'}) parties = ';'.join([ span.text.strip() for span in tr.find_all( 'span', { 'class': '_onkszavazokorieredmenyek_WAR_nvinvrportlet_popover' }) ]) l1.append(parties) l2.append(int(columns[0].text)) df_county = pd.DataFrame([l1, l2]) return df_county else: return None
def GetPersonalTable(url): raw_html = webfunctions.simple_get(url) html = BeautifulSoup(raw_html, 'html.parser') national_link = html.findAll('button')[2].attrs['onclick'][15:-1] buttons = html.findAll('a') detailed_link = [x for x in buttons if 'A szavazókör általános adatai' in x.text][0].attrs['href'] table = html.find(lambda tag: tag.name == 'table') if table is None: return [], [], [], [] else: rows = table.findAll(lambda tag: tag.name == 'tr') l1 = [] l2 = [] l3 = [] for tr in rows[1:]: if len(tr.find_all('td')) > 1: l1.append(tr.find_all('td')[1].find_all('div')[1].find_all('a')[0].text) l2.append(tr.find_all('td')[1].find_all('div')[1].find_all('div')[0].text) l3.append(tr.find_all('td')[2].find_all('div')[0].text) df = pd.DataFrame([l1, l2, l3]) table = html.findAll("div", {"class": "nvi-summary-container summary-container-ogy first-container nvi-collapsed not-collapsable"}) if len(table) == 0: table = html.findAll("div", {"class": "nvi-summary-container summary-container-ogy first-container nvi-collapsed"}) if len(table[0].find_all('span')) > 11: nszvsz = table[0].find_all('span')[0].text if table[0].find_all('div')[14].text == ' Megjelent ': m = table[0].find_all('div')[15].text else: m = table[0].find_all('div')[14].text if table[0].find_all('div')[14].text == ' Megjelent ': nsz = table[0].find_all('div')[20].text else: nsz = table[0].find_all('div')[19].text ulbnszsz = table[0].find_all('span')[5].text ulbszsz = table[0].find_all('span')[7].text easzmsz = table[0].find_all('span')[9].text elszsz = table[0].find_all('span')[11].text eszsz = table[0].find_all('span')[13].text else: othertable = html.findAll("div", {"class": "nvi-summary-content summary-content-kijeloltJkv toggler-content-collapsed hide"}) nszvsz = table[0].find_all('span')[0].text m = othertable[0].find_all('div')[13].text nsz = othertable[0].find_all('div')[18].text ulbnszsz = 0 ulbszsz = othertable[0].find_all('div')[22].findAll('span')[2].text easzmsz = 0 elszsz = othertable[0].find_all('div')[23].findAll('span')[2].text eszsz = othertable[0].find_all('div')[24].findAll('span')[2].text results = [nszvsz, m, nsz, ulbnszsz, ulbszsz, easzmsz, elszsz, eszsz] return df, results, national_link, detailed_link
def GetNationalTable(url): raw_html = webfunctions.simple_get(url) html = BeautifulSoup(raw_html, 'html.parser') table = html.findAll("div", {"class": "nvi-search-list"}) l1 = [] l2 = [] for i in range(0, len(table[0].find_all('div', {"class": "span7"}))): l1.append(table[0].find_all('div', {"class": "span7"})[i].text) l2.append(table[0].find_all('div', {"class": "span6 text-right"})[i].text) df = pd.DataFrame([l1, l2]) return df
def getdetails(url): raw_html = webfunctions.simple_get(url) html = BeautifulSoup(raw_html, 'html.parser') table = html.find(lambda tag: tag.name == 'table') if table is not None: rows = table.findAll(lambda tag: tag.name == 'tr') l1 = [] l2 = [] for tr in rows: l1.append(tr.find_all('td')[0].find_all('div')[1].text) l2.append(tr.find_all('td')[0].find_all('div')[2].text) df = pd.DataFrame([l1, l2]) else: l1 = ['egész város'] l2 = ['egész város'] df = pd.DataFrame([l1, l2]) return df
def GetLinksOfRegion(url): raw_html = webfunctions.simple_get(url) html = BeautifulSoup(raw_html, 'html.parser') table = html.findAll("a") table = [x for x in table if hasattr(x, 'attrs')] table = [x for x in table if 'href' in x.attrs] table = [x for x in table if 'szavkorSorszam' in x.attrs['href']] table = [x for x in table if 'Szavazóhelyiség címe' not in x.text] href_list = [x.attrs['href'] for x in table] name = [x.text for x in table] oevk = html.findAll("div", {"class": "span6"}) oevk = [x.text for x in oevk if 'számú OEVK' in x.text] return href_list, name, oevk
def getnationalresults(url): raw_html = webfunctions.simple_get(url) html = BeautifulSoup(raw_html, 'html.parser') table = html.findAll("div", {"class": "nvi-search-list"}) l1 = [] l2 = [] if not table: return None, None else: for i in range(0, len(table[0].find_all('div', {"class": "span4 text-right"}))): l2.append(int(table[0].find_all('div', {"class": "span4 text-right"})[i].text[:table[0].find_all('div', {"class": "span4 text-right"})[i].text.find('(')].replace('\xa0', '').replace(' ', ''))) for i in range(0, len(table[0].find_all('div', {"class": "span8"}))): if table[0].find_all('div', {"class": "span8"})[i].text != ' Érvényes szavazatok (%): ': l1.append(table[0].find_all('div', {"class": "span8"})[i].text.strip()) df_national = pd.DataFrame([l1, l2]) participation = {} table = html.findAll("div", {"class": "nvi-summary-content summary-content-fejadatok toggler-content-collapsed hide"}) # if len(table) == 0: # table = html.findAll("div", {"class": "nvi-summary-container summary-container-ogy first-container nvi-collapsed"}) if len(table[0].find_all('span')) > 11: participation['nszvsz'] = int(table[0].find_all('span')[0].text.replace('\xa0', '').replace(' fő. ', '')) if table[0].find_all('div')[14].text == ' Megjelent ': participation['mj'] = int(table[0].find_all('div')[17].text[:table[0].find_all('div')[17].text.find('fő')].replace('Megjelent', '').replace('\xa0', '').replace(' ', '')) else: participation['mj'] = int(table[0].find_all('div')[16].text[:table[0].find_all('div')[16].text.find('fő')].replace('Megjelent', '').replace('\xa0', '').replace(' ', '')) if table[0].find_all('div')[14].text == ' Megjelent ': participation['nsz'] = int(table[0].find_all('div')[20].text[:table[0].find_all('div')[20].text.find('fő')].replace('Nem szavazott', '').replace('\xa0', '').replace(' ', '')) else: participation['nsz'] = int(table[0].find_all('div')[19].text[:table[0].find_all('div')[19].text.find('fő')].replace('Nem szavazott', '').replace('\xa0', '').replace(' ', '')) participation['ulbnszsz'] = int(table[0].find_all('span')[5].text.replace('\xa0', '')) participation['ulbszsz'] = int(table[0].find_all('span')[7].text.replace('\xa0', '')) participation['easzmsz'] = int(table[0].find_all('span')[9].text.replace('\xa0', '')) participation['elszsz'] = int(table[0].find_all('span')[11].text.replace('\xa0', '')) participation['eszsz'] = int(table[0].find_all('span')[13].text.replace('\xa0', '')) else: othertable = html.findAll("div", {"class": "nvi-summary-content summary-content-kijeloltJkv toggler-content-collapsed hide"}) nszvsz = int(table[0].find_all('span')[0].text.replace('\xa0', '').replace(' fő. ', '')) m = int(othertable[0].find_all('div')[13].text) nsz = int(othertable[0].find_all('div')[18].text) ulbnszsz = 0 ulbszsz = int(othertable[0].find_all('div')[22].findAll('span')[2].text) easzmsz = 0 elszsz = int(othertable[0].find_all('div')[23].findAll('span')[2].text) eszsz = int(othertable[0].find_all('div')[24].findAll('span')[2].text) return df_national, participation
def getbatchdistrict(maz, taz, idx): maz = padtolen(maz, 2) taz = padtolen(taz, 3) url = 'https://portal.valasztas.hu/szavazokorok_ep2019?p_p_lifecycle=0&p_p_state=maximized&p_p_mode=view&p_p_id=epszavazokorok_WAR_nvinvrportlet&_epszavazokorok_WAR_nvinvrportlet_searchSortColumn=SORSZAM&_epszavazokorok_WAR_nvinvrportlet_searchText=Debrecen+%28Hajd%C3%BA-Bihar%29&_epszavazokorok_WAR_nvinvrportlet_telepulesKod=null&_epszavazokorok_WAR_nvinvrportlet_megyeKod2=' + maz + '&_epszavazokorok_WAR_nvinvrportlet_megyeKod=null&_epszavazokorok_WAR_nvinvrportlet_vlId=291&_epszavazokorok_WAR_nvinvrportlet_searchSortType=asc&_epszavazokorok_WAR_nvinvrportlet_vltId=684&_epszavazokorok_WAR_nvinvrportlet_telepulesKod2=' + taz + '&_epszavazokorok_WAR_nvinvrportlet_valasztasTipusKod=E&_epszavazokorok_WAR_nvinvrportlet_delta=200&_epszavazokorok_WAR_nvinvrportlet_keywords=&_epszavazokorok_WAR_nvinvrportlet_advancedSearch=false&_epszavazokorok_WAR_nvinvrportlet_andOperator=true&_epszavazokorok_WAR_nvinvrportlet_resetCur=false&_epszavazokorok_WAR_nvinvrportlet_cur=' + str(idx) raw_html = webfunctions.simple_get(url) html = BeautifulSoup(raw_html, 'html.parser') table = html.find(lambda tag: tag.name == 'table') if table: city = html.findAll('h2')[2].text.strip() rows = table.findAll(lambda tag: tag.name == 'tr') district = [] for row in rows: attrs = {} columns = row.findAll(lambda tag: tag.name == 'td') if columns[0].find_all('a'): atext = columns[0].find_all('a')[0].text atext = atext.replace('Szavazóhelyiség címe:', '') atext = atext.strip() atext = atext.split(' ', 1)[0] href = columns[0].find_all('a')[0].attrs['href'] attrs['href'] = href attrs['maz'] = int(maz) attrs['taz'] = int(re.search('WAR_nvinvrportlet_telepulesKod=(.+?)&_epszavazokorieredmenyek_WAR_nvinvrportlet', href).group(1)) attrs['sorsz'] = int(re.search('_epszavazokorieredmenyek_WAR_nvinvrportlet_szavkorSorszam=(.*)', href).group(1)) attrs['cimk'] = columns[0].find_all('a')[0].find_all('div')[0].text if len(columns[0].find_all('a')[0].find_all('div')) > 1: attrs['cimk'] = attrs['cimk'] + columns[0].find_all('a')[0].find_all('div')[1].text attrs['cimt'] = city attrs['evk'] = 0 attrs['tip'] = 0 if district: if attrs['maz'] != district[-1]['maz'] or attrs['taz'] != district[-1]['taz'] or attrs['sorsz'] != district[-1]['sorsz']: district.append(attrs) else: district.append(attrs) return district else: return None
from bs4 import BeautifulSoup from processing import webfunctions from subroutes import subtables from storage import db_scripts i_from = 11 j_from = 1 idx_from = 0 if i_from < 2: for i in range(i_from, 2): for j in range(j_from, 21): county = str(i) oevk = str(j) url = 'https://www.valasztas.hu/szavazokori-eredmenyek?_szavazokorok_WAR_nvinvrportlet_formDate=32503680000000&p_p_id=szavazokorok_WAR_nvinvrportlet&p_p_lifecycle=0&p_p_state=normal&p_p_mode=view&p_p_col_id=column-2&p_p_col_pos=1&p_p_col_count=2&_szavazokorok_WAR_nvinvrportlet_vlId=244&_szavazokorok_WAR_nvinvrportlet_vltId=556&_szavazokorok_WAR_nvinvrportlet_megyeKod=&_szavazokorok_WAR_nvinvrportlet_telepulesKod=&_szavazokorok_WAR_nvinvrportlet_searchSortColumn=&_szavazokorok_WAR_nvinvrportlet_searchSortType=asc&_szavazokorok_WAR_nvinvrportlet_wardClean=false&_szavazokorok_WAR_nvinvrportlet_wardSettlement=false&_szavazokorok_WAR_nvinvrportlet_megyeKod2=01&_szavazokorok_WAR_nvinvrportlet_telepulesKod2=&_szavazokorok_WAR_nvinvrportlet_valasztasTipusKod=&_szavazokorok_WAR_nvinvrportlet_evkSzam=&_szavazokorok_WAR_nvinvrportlet_szavkorTypes=&_szavazokorok_WAR_nvinvrportlet_valasztasIntegralt=false&_szavazokorok_WAR_nvinvrportlet_oevkKod=' + oevk + '&_szavazokorok_WAR_nvinvrportlet_searchText=Budapest+1.+sz%C3%A1m%C3%BA+OEVK&_szavazokorok_WAR_nvinvrportlet_delta=2000' raw_html = webfunctions.simple_get(url) html = BeautifulSoup(raw_html, 'html.parser') oevk = html.findAll("div", {"class": "span6"}) oevk = [x for x in oevk if 'számú OEVK' in x.text] print(i) print(j) if len(oevk) > 0: href_list, name, oevk = subtables.GetLinksOfRegion(url) if i == i_from and j == j_from: for idx in range(idx_from, len(href_list)): district_id = db_scripts.insert_district( i, j, name[idx], oevk[idx], idx) href = href_list[idx] df, results, national_link, detailed_link = subtables.GetPersonalTable(
def getmayorresults(url): raw_html = webfunctions.simple_get(url) html = BeautifulSoup(raw_html, 'html.parser') table = html.find(lambda tag: tag.name == 'table') if not table: return None, None else: rows = table.findAll(lambda tag: tag.name == 'tr') l1 = [] l2 = [] l3 = [] for tr in rows: columns = tr.find_all('span') l1.append(columns[0].text.strip()) l3.append( int( re.search('(.*)\(', columns[2].text.strip()).group(0)[0:-2])) parties = ';'.join([ span.text.strip() for span in tr.find_all('span', { 'class': '_onkszavazokorieredmenyek_WAR_nvinvrportlet_popover' }) ]) l2.append(parties) df_mayor = pd.DataFrame([l1, l2, l3]) participation = {} table = html.findAll( "div", { "class": "nvi-summary-content summary-content-fejadatok toggler-content-collapsed hide" }) # if len(table) == 0: # table = html.findAll("div", {"class": "nvi-summary-container summary-container-ogy first-container nvi-collapsed"}) if len(table[0].find_all('span')) > 11: participation['nszvsz'] = int( table[0].find_all('span')[0].text.replace('\xa0', '').replace( ' fő', '')) if table[0].find_all('div')[14].text == ' Megjelent ': participation['mj'] = int(table[0].find_all( 'div')[17].text[:table[0].find_all('div')[17].text. find('fő')].replace( 'Megjelent', '').replace('\xa0', '').replace(' ', '')) else: participation['mj'] = int(table[0].find_all( 'div')[16].text[:table[0].find_all('div')[16].text. find('fő')].replace( 'Megjelent', '').replace('\xa0', '').replace(' ', '')) if table[0].find_all('div')[14].text == ' Megjelent ': participation['nsz'] = int(table[0].find_all( 'div')[20].text[:table[0].find_all('div')[20]. text.find('fő')].replace( 'Nem szavazott', '').replace('\xa0', '').replace(' ', '')) else: participation['nsz'] = int(table[0].find_all( 'div')[19].text[:table[0].find_all('div')[19]. text.find('fő')].replace( 'Nem szavazott', '').replace('\xa0', '').replace(' ', '')) participation['ulbnszsz'] = int( table[0].find_all('span')[5].text.replace('\xa0', '')) participation['ulbszsz'] = int( table[0].find_all('span')[7].text.replace('\xa0', '')) participation['easzmsz'] = int( table[0].find_all('span')[9].text.replace('\xa0', '')) participation['elszsz'] = int( table[0].find_all('span')[11].text.replace('\xa0', '')) participation['eszsz'] = int( table[0].find_all('span')[13].text.replace('\xa0', '')) else: othertable = html.findAll( "div", { "class": "nvi-summary-content summary-content-kijeloltJkv toggler-content-collapsed hide" }) nszvsz = int(table[0].find_all('span')[0].text.replace( '\xa0', '').replace(' fő. ', '')) m = int(othertable[0].find_all('div')[13].text) nsz = int(othertable[0].find_all('div')[18].text) ulbnszsz = 0 ulbszsz = int( othertable[0].find_all('div')[22].findAll('span')[2].text) easzmsz = 0 elszsz = int( othertable[0].find_all('div')[23].findAll('span')[2].text) eszsz = int( othertable[0].find_all('div')[24].findAll('span')[2].text) return df_mayor, participation