def test_import_from_ods_retrieve_desired_data(self, mocked_create_table):
mocked_create_table.return_value = 42
# import using filename
rows.import_from_ods(self.filename)
call_args = mocked_create_table.call_args_list[0]
self.assert_create_table_data(call_args)
# import using fobj
with open(self.filename, "rb") as fobj:
rows.import_from_ods(fobj)
call_args = mocked_create_table.call_args_list[1]
self.assert_create_table_data(call_args)
def extract_ativos_2018(self):
fields_ativos_2018 = OrderedDict([
("matricula", rows.fields.IntegerField),
("nome", rows.fields.TextField),
("cargo", rows.fields.TextField),
("lotacao", rows.fields.TextField),
("remuneracao_cargo_efetivo", rows.fields.DecimalField),
("remuneracao_outras", rows.fields.DecimalField),
("remuneracao_funcao_confianca_ou_cc", rows.fields.DecimalField),
("remuneracao_13o_salario", rows.fields.DecimalField),
("remuneracao_ferias", rows.fields.DecimalField),
("remuneracao_abono_de_permanencia", rows.fields.DecimalField),
("rendimento_bruto", rows.fields.DecimalField),
("desconto_contribuicao_previdenciaria", rows.fields.DecimalField),
("desconto_imposto_de_renda", rows.fields.DecimalField),
("desconto_retencao_teto", rows.fields.DecimalField),
("desconto_total", rows.fields.DecimalField),
("rendimento_liquido", rows.fields.DecimalField),
("auxilio_alimentacao", rows.fields.DecimalField),
("auxilio_transporte", rows.fields.DecimalField),
("auxilio_creche", rows.fields.DecimalField),
("auxilio_ferias_em_pecunia", rows.fields.DecimalField),
("remuneracao_temporaria_outras", rows.fields.DecimalField),
])
table = rows.import_from_ods(self.filename,
start_row=3,
skip_header=False,
fields=fields_ativos_2018)
for row in table:
yield self.convert_row(row)
def test_issue_290_one_hundred_read_as_1(self):
result = rows.import_from_ods("tests/data/text_in_percent_cell.ods")
# As this test is written, file numeric file contents on first column are
# 100%, 23.20%, 1.00%, 10.00%, 100.00%
assert result[0][0] == Decimal("1")
assert result[2][0] == Decimal("0.01")
assert result[3][0] == Decimal("0.1")
assert result[4][0] == Decimal("1")
def abre_doc(self):
if self.doc_tipo == 'xls':
self.doc = rows.import_from_xls(self.doc)
elif self.doc_tipo == 'xlsx':
self.doc = rows.import_from_xlsx(self.doc)
elif self.doc_tipo == 'ods':
self.doc = rows.import_from_ods(self.doc)
else:
raise Exception('Erro ao abrir o arquivo.')
def test_import_from_ods_uses_create_table(self, mocked_create_table):
mocked_create_table.return_value = 42
kwargs = {"encoding": "test", "some_key": 123, "other": 456}
result = rows.import_from_ods(self.filename, **kwargs)
self.assertTrue(mocked_create_table.called)
self.assertEqual(mocked_create_table.call_count, 1)
self.assertEqual(result, 42)
call = mocked_create_table.call_args
kwargs["meta"] = {"imported_from": "ods", "filename": self.filename}
self.assertEqual(call[1], kwargs)
def test_import_from_ods_uses_create_table(self, mocked_create_table):
mocked_create_table.return_value = 42
kwargs = {'encoding': 'test', 'some_key': 123, 'other': 456, }
result = rows.import_from_ods(self.filename, **kwargs)
self.assertTrue(mocked_create_table.called)
self.assertEqual(mocked_create_table.call_count, 1)
self.assertEqual(result, 42)
call = mocked_create_table.call_args
kwargs['meta'] = {'imported_from': 'ods', 'filename': self.filename, }
self.assertEqual(call[1], kwargs)
def test_import_from_ods_uses_create_table(self, mocked_create_table):
mocked_create_table.return_value = 42
kwargs = {'encoding': 'test', 'some_key': 123, 'other': 456, }
result = rows.import_from_ods(self.filename, **kwargs)
self.assertTrue(mocked_create_table.called)
self.assertEqual(mocked_create_table.call_count, 1)
self.assertEqual(result, 42)
call = mocked_create_table.call_args
kwargs['meta'] = {'imported_from': 'ods', 'filename': self.filename, }
self.assertEqual(call[1], kwargs)
def extract(self):
filename, metadata = self.filename, self.metadata
start_row, fields = None, None
# TODO: remuneraçãodeexeciciosanteriores_ativos
# TODO: remuneraçãodeexeciciosanteriores_inativos
# TODO: verbasindenizatorias_ativos
# TODO: verbasindenizatorias_inativos
year_month = int(f"{metadata['ano']}{metadata['mes']:02d}")
observation = metadata["observacao"]
if len(observation.replace("_ativos", "")) == 4:
if year_month >= 201_708:
start_row = 3
fields = self.fields_2018_ativos
elif year_month >= 201_603:
start_row = 1
fields = self.fields_2016_ativos_e_inativos
elif year_month == 201_602:
start_row = 1
fields = self.fields_2016_ativos_excecao
elif year_month >= 201_511:
start_row = 2
fields = self.fields_2015_ativos_e_inativos
elif len(observation.replace("_inativos", "")) == 4:
if year_month >= 201_708:
start_row = 3
fields = self.fields_2018_inativos
elif year_month >= 201_602:
start_row = 1
fields = self.fields_2016_ativos_e_inativos
elif year_month >= 201_511:
start_row = 2
fields = self.fields_2015_ativos_e_inativos
if start_row and fields:
table = rows.import_from_ods(self.filename,
start_row=start_row,
fields=fields)
for row in table:
if "registros listados" in row.nome.lower():
break
yield self.convert_row(row._asdict())
def extract_inativos_2018(self):
fields_inativos_2018 = OrderedDict([
("matricula", rows.fields.IntegerField),
("nome", rows.fields.TextField),
("cargo", rows.fields.TextField),
("lotacao", rows.fields.TextField),
("remuneracao_cargo_efetivo", rows.fields.DecimalField),
("remuneracao_outras", rows.fields.DecimalField),
("remuneracao_13o_salario", rows.fields.DecimalField),
("rendimento_bruto", rows.fields.DecimalField),
("desconto_contribuicao_previdenciaria", rows.fields.DecimalField),
("desconto_imposto_de_renda", rows.fields.DecimalField),
("desconto_total", rows.fields.DecimalField),
("rendimento_liquido", rows.fields.DecimalField),
])
table = rows.import_from_ods(self.filename,
start_row=3,
skip_header=False,
fields=fields_inativos_2018)
for row in table:
yield self.convert_row(row)
def extract(self):
filename, metadata = self.filename, self.metadata
# From 2017-11 to 2018-12, get data from Brasil.IO (in CSV) using the
# `extract_magistrados` helper function.
if "contracheque.csv" in filename.name:
yield from extract_magistrados(filename, self.state)
else:
# TODO: check repeated months
if metadata["ano"] == 2018 or (metadata["ano"] == 2017
and metadata["mes"] in (11, 12)):
# Data already converted in contracheque.csv
return
extension = filename.name.split(".")[-1].lower()
if extension == "xls":
table = rows.import_from_xls(filename, start_row=3)
for row in table:
yield {
"cargo": row.cargo_no_orgao,
"nome": row.nome,
"rendimento_bruto": row.total_decreditos_v,
"rendimento_liquido": row.rendimentoliquido_xi,
}
elif extension == "ods":
fields_1 = OrderedDict([
("nome", rows.fields.TextField),
("lotacao", rows.fields.TextField),
("cargo_no_orgao", rows.fields.TextField),
("remuneracao_paradigma", MoneyField),
("vantagens_pessoais", MoneyField),
("subsidio_diferenca_de_subsidio_funcao_de_confianca_ou_cargo_em_comissao",
MoneyField),
("auxilios_indenizacoes", MoneyField),
("auxilios_indenizacoes_pagto_tesouraria", MoneyField),
("vantagens_eventuais", MoneyField),
("decimo_terceiro_adiantamento2a_parcela", MoneyField),
("terco_constitucional_de_ferias", MoneyField),
("pensao_provisoria_de_montepio", MoneyField),
("total_de_creditos", MoneyField),
("previdencia_publica", MoneyField),
("imposto_de_renda", MoneyField),
("descontos_diversos", MoneyField),
("retencao_por_teto_constitucional", MoneyField),
("total_de_descontos_debitos", MoneyField),
("rendimento_liquido", MoneyField),
("remuneracao_do_orgao_de_origem", MoneyField),
("diarias", MoneyField),
])
fields_2 = OrderedDict([
("matricula", rows.fields.TextField),
("nome", rows.fields.TextField),
("lotacao", rows.fields.TextField),
("cargo_no_orgao", rows.fields.TextField),
("remuneracao_paradigma", MoneyField),
("vantagens_pessoais", MoneyField),
("subsidio_diferenca_de_subsidio_funcao_de_confianca_ou_cargo_em_comissao",
MoneyField),
("auxilios_indenizacoes", MoneyField),
("auxilios_indenizacoes_pagto_tesouraria", MoneyField),
("vantagens_eventuais", MoneyField),
("decimo_terceiro_adiantamento2a_parcela", MoneyField),
("terco_constitucional_de_ferias", MoneyField),
("pensao_provisoria_de_montepio", MoneyField),
("total_de_creditos", MoneyField),
("previdencia_publica", MoneyField),
("imposto_de_renda", MoneyField),
("descontos_diversos", MoneyField),
("retencao_por_teto_constitucional", MoneyField),
("total_de_descontos_debitos", MoneyField),
("rendimento_liquido", MoneyField),
("remuneracao_do_orgao_de_origem", MoneyField),
("diarias", MoneyField),
])
import decimal
try:
table = rows.import_from_ods(filename,
start_row=3,
end_column=20,
skip_header=True,
fields=fields_1)
except (ValueError,
decimal.InvalidOperation): # "Matricula" is hidden
return # TODO: fix this case
else:
for row in table:
yield {
"nome": row.nome,
"cargo": row.cargo_no_orgao,
"rendimento_liquido": row.rendimento_liquido,
"rendimento_bruto": row.total_de_creditos,
}
elif extension == "pdf":
return
def extract_mpe_sp(self):
table = rows.import_from_ods(self.filename,
start_row=0,
skip_header=False)
for row in table:
yield self.convert_row(row)