def test_import_from_ods_retrieve_desired_data(self, mocked_create_table): mocked_create_table.return_value = 42 # import using filename rows.import_from_ods(self.filename) call_args = mocked_create_table.call_args_list[0] self.assert_create_table_data(call_args) # import using fobj with open(self.filename, "rb") as fobj: rows.import_from_ods(fobj) call_args = mocked_create_table.call_args_list[1] self.assert_create_table_data(call_args)
def extract_ativos_2018(self): fields_ativos_2018 = OrderedDict([ ("matricula", rows.fields.IntegerField), ("nome", rows.fields.TextField), ("cargo", rows.fields.TextField), ("lotacao", rows.fields.TextField), ("remuneracao_cargo_efetivo", rows.fields.DecimalField), ("remuneracao_outras", rows.fields.DecimalField), ("remuneracao_funcao_confianca_ou_cc", rows.fields.DecimalField), ("remuneracao_13o_salario", rows.fields.DecimalField), ("remuneracao_ferias", rows.fields.DecimalField), ("remuneracao_abono_de_permanencia", rows.fields.DecimalField), ("rendimento_bruto", rows.fields.DecimalField), ("desconto_contribuicao_previdenciaria", rows.fields.DecimalField), ("desconto_imposto_de_renda", rows.fields.DecimalField), ("desconto_retencao_teto", rows.fields.DecimalField), ("desconto_total", rows.fields.DecimalField), ("rendimento_liquido", rows.fields.DecimalField), ("auxilio_alimentacao", rows.fields.DecimalField), ("auxilio_transporte", rows.fields.DecimalField), ("auxilio_creche", rows.fields.DecimalField), ("auxilio_ferias_em_pecunia", rows.fields.DecimalField), ("remuneracao_temporaria_outras", rows.fields.DecimalField), ]) table = rows.import_from_ods(self.filename, start_row=3, skip_header=False, fields=fields_ativos_2018) for row in table: yield self.convert_row(row)
def test_issue_290_one_hundred_read_as_1(self): result = rows.import_from_ods("tests/data/text_in_percent_cell.ods") # As this test is written, file numeric file contents on first column are # 100%, 23.20%, 1.00%, 10.00%, 100.00% assert result[0][0] == Decimal("1") assert result[2][0] == Decimal("0.01") assert result[3][0] == Decimal("0.1") assert result[4][0] == Decimal("1")
def abre_doc(self): if self.doc_tipo == 'xls': self.doc = rows.import_from_xls(self.doc) elif self.doc_tipo == 'xlsx': self.doc = rows.import_from_xlsx(self.doc) elif self.doc_tipo == 'ods': self.doc = rows.import_from_ods(self.doc) else: raise Exception('Erro ao abrir o arquivo.')
def test_import_from_ods_uses_create_table(self, mocked_create_table): mocked_create_table.return_value = 42 kwargs = {"encoding": "test", "some_key": 123, "other": 456} result = rows.import_from_ods(self.filename, **kwargs) self.assertTrue(mocked_create_table.called) self.assertEqual(mocked_create_table.call_count, 1) self.assertEqual(result, 42) call = mocked_create_table.call_args kwargs["meta"] = {"imported_from": "ods", "filename": self.filename} self.assertEqual(call[1], kwargs)
def test_import_from_ods_uses_create_table(self, mocked_create_table): mocked_create_table.return_value = 42 kwargs = {'encoding': 'test', 'some_key': 123, 'other': 456, } result = rows.import_from_ods(self.filename, **kwargs) self.assertTrue(mocked_create_table.called) self.assertEqual(mocked_create_table.call_count, 1) self.assertEqual(result, 42) call = mocked_create_table.call_args kwargs['meta'] = {'imported_from': 'ods', 'filename': self.filename, } self.assertEqual(call[1], kwargs)
def extract(self): filename, metadata = self.filename, self.metadata start_row, fields = None, None # TODO: remuneraçãodeexeciciosanteriores_ativos # TODO: remuneraçãodeexeciciosanteriores_inativos # TODO: verbasindenizatorias_ativos # TODO: verbasindenizatorias_inativos year_month = int(f"{metadata['ano']}{metadata['mes']:02d}") observation = metadata["observacao"] if len(observation.replace("_ativos", "")) == 4: if year_month >= 201_708: start_row = 3 fields = self.fields_2018_ativos elif year_month >= 201_603: start_row = 1 fields = self.fields_2016_ativos_e_inativos elif year_month == 201_602: start_row = 1 fields = self.fields_2016_ativos_excecao elif year_month >= 201_511: start_row = 2 fields = self.fields_2015_ativos_e_inativos elif len(observation.replace("_inativos", "")) == 4: if year_month >= 201_708: start_row = 3 fields = self.fields_2018_inativos elif year_month >= 201_602: start_row = 1 fields = self.fields_2016_ativos_e_inativos elif year_month >= 201_511: start_row = 2 fields = self.fields_2015_ativos_e_inativos if start_row and fields: table = rows.import_from_ods(self.filename, start_row=start_row, fields=fields) for row in table: if "registros listados" in row.nome.lower(): break yield self.convert_row(row._asdict())
def extract_inativos_2018(self): fields_inativos_2018 = OrderedDict([ ("matricula", rows.fields.IntegerField), ("nome", rows.fields.TextField), ("cargo", rows.fields.TextField), ("lotacao", rows.fields.TextField), ("remuneracao_cargo_efetivo", rows.fields.DecimalField), ("remuneracao_outras", rows.fields.DecimalField), ("remuneracao_13o_salario", rows.fields.DecimalField), ("rendimento_bruto", rows.fields.DecimalField), ("desconto_contribuicao_previdenciaria", rows.fields.DecimalField), ("desconto_imposto_de_renda", rows.fields.DecimalField), ("desconto_total", rows.fields.DecimalField), ("rendimento_liquido", rows.fields.DecimalField), ]) table = rows.import_from_ods(self.filename, start_row=3, skip_header=False, fields=fields_inativos_2018) for row in table: yield self.convert_row(row)
def extract(self): filename, metadata = self.filename, self.metadata # From 2017-11 to 2018-12, get data from Brasil.IO (in CSV) using the # `extract_magistrados` helper function. if "contracheque.csv" in filename.name: yield from extract_magistrados(filename, self.state) else: # TODO: check repeated months if metadata["ano"] == 2018 or (metadata["ano"] == 2017 and metadata["mes"] in (11, 12)): # Data already converted in contracheque.csv return extension = filename.name.split(".")[-1].lower() if extension == "xls": table = rows.import_from_xls(filename, start_row=3) for row in table: yield { "cargo": row.cargo_no_orgao, "nome": row.nome, "rendimento_bruto": row.total_decreditos_v, "rendimento_liquido": row.rendimentoliquido_xi, } elif extension == "ods": fields_1 = OrderedDict([ ("nome", rows.fields.TextField), ("lotacao", rows.fields.TextField), ("cargo_no_orgao", rows.fields.TextField), ("remuneracao_paradigma", MoneyField), ("vantagens_pessoais", MoneyField), ("subsidio_diferenca_de_subsidio_funcao_de_confianca_ou_cargo_em_comissao", MoneyField), ("auxilios_indenizacoes", MoneyField), ("auxilios_indenizacoes_pagto_tesouraria", MoneyField), ("vantagens_eventuais", MoneyField), ("decimo_terceiro_adiantamento2a_parcela", MoneyField), ("terco_constitucional_de_ferias", MoneyField), ("pensao_provisoria_de_montepio", MoneyField), ("total_de_creditos", MoneyField), ("previdencia_publica", MoneyField), ("imposto_de_renda", MoneyField), ("descontos_diversos", MoneyField), ("retencao_por_teto_constitucional", MoneyField), ("total_de_descontos_debitos", MoneyField), ("rendimento_liquido", MoneyField), ("remuneracao_do_orgao_de_origem", MoneyField), ("diarias", MoneyField), ]) fields_2 = OrderedDict([ ("matricula", rows.fields.TextField), ("nome", rows.fields.TextField), ("lotacao", rows.fields.TextField), ("cargo_no_orgao", rows.fields.TextField), ("remuneracao_paradigma", MoneyField), ("vantagens_pessoais", MoneyField), ("subsidio_diferenca_de_subsidio_funcao_de_confianca_ou_cargo_em_comissao", MoneyField), ("auxilios_indenizacoes", MoneyField), ("auxilios_indenizacoes_pagto_tesouraria", MoneyField), ("vantagens_eventuais", MoneyField), ("decimo_terceiro_adiantamento2a_parcela", MoneyField), ("terco_constitucional_de_ferias", MoneyField), ("pensao_provisoria_de_montepio", MoneyField), ("total_de_creditos", MoneyField), ("previdencia_publica", MoneyField), ("imposto_de_renda", MoneyField), ("descontos_diversos", MoneyField), ("retencao_por_teto_constitucional", MoneyField), ("total_de_descontos_debitos", MoneyField), ("rendimento_liquido", MoneyField), ("remuneracao_do_orgao_de_origem", MoneyField), ("diarias", MoneyField), ]) import decimal try: table = rows.import_from_ods(filename, start_row=3, end_column=20, skip_header=True, fields=fields_1) except (ValueError, decimal.InvalidOperation): # "Matricula" is hidden return # TODO: fix this case else: for row in table: yield { "nome": row.nome, "cargo": row.cargo_no_orgao, "rendimento_liquido": row.rendimento_liquido, "rendimento_bruto": row.total_de_creditos, } elif extension == "pdf": return
def extract_mpe_sp(self): table = rows.import_from_ods(self.filename, start_row=0, skip_header=False) for row in table: yield self.convert_row(row)