def pegar_dados_da_camara_municipal(id_ver, row_ver):
response = requests.get(URL_CAMARA.format(id=id_ver))
html = response.content
if not response.status_code == requests.codes.OK:
raise ConnectionError(u'An error was occoured while trying to get id: {}'.format(id_ver))
soup = BeautifulSoup(html, 'html.parser')
div_dados = soup.find(attrs={"class": "vereainterna"})
itens_dados = list(div_dados.find('ul').find_all('li'))
def parse_data(data):
link_regex = '<a [^>]*>(.+?)</a>'
link_pattern = re.compile(link_regex)
match = link_pattern.match(data)
if match:
return match.group(1)
else:
return data
for item in itens_dados:
match = pattern.match(str(item))
if match:
row_ver[fields.slug(match.group(1))] = parse_data(match.group(3))
row_ver["frentes"] = URL_FRENTES.format(id=id_ver)
row_ver["votacoes"] = URL_VOTACOES.format(id=id_ver)
row_ver["mandatos"] = URL_MANDATOS.format(id=id_ver)
def replace_names(a):
return (slug(fix_tribunal(a)).replace(
"vantagens_",
"direitos_").replace("trt_",
"tribunal_regional_do_trabalho_").replace(
"trf_",
"tribunal_regional_federal_").replace(
"justica_federal_",
"tribunal_regional_federal_"))
def read_header(filename):
"""Read a CSV file which describes a fixed-width file
The CSV must have the following columns:
- name (final field name)
- size (fixed size of the field, in bytes)
- start_column (column in the fwf file where the fields starts)
- type ("A" for text, "N" for int)
"""
table = rows.import_from_csv(filename)
table.order_by("start_column")
header = []
for row in table:
row = dict(row._asdict())
row["field_name"] = slug(row["name"])
row["start_index"] = row["start_column"] - 1
row["end_index"] = row["start_index"] + row["size"]
header.append(row)
return header