Esempio n. 1
0
    def post(self, request, *args, **kwargs):
        form_class = self.get_form_class()
        form = self.get_form(form_class)

        if form.is_valid():
            data = form.cleaned_data

            # When using a model form, you must use the
            # name attribute of the file rather than
            # passing the request file var directly as this is the
            # required when using the chunk uploader project
            s3_file_name = request.FILES['file'].name

            file_upload = FileUpload(
                s3_document_file=s3_file_name,
                uploading_user=request.user,
                document_type=FileUpload.BUDGET,
            )
            file_upload.save()
            # Process file async

            if settings.ASYNC_FILE_UPLOAD:
                process_uploaded_file.delay(data['year'].financial_year, )
            else:
                process_uploaded_file(data['year'].financial_year, )

            return self.form_valid(form)
        else:
            return self.form_invalid(form)
Esempio n. 2
0
    def handle(self, *args, **options):
        path = options["path"]
        year = options["financial_year"]

        file_name = self.path_to_upload(path, 'xlsx')
        fileobj = FileUpload(
            document_file_name=file_name,
            document_type=FileUpload.PREVIOUSYEAR,
            file_location=FileUpload.LOCALFILE,
        )
        fileobj.save()

        try:
            upload_previous_year_from_file(fileobj, year)
        except (WrongChartOFAccountCodeException, WrongArchivePeriodException,
                UploadFileDataError, UploadFileFormatError,
                ArchiveYearError) as ex:
            raise CommandError(
                f"Failure uploading historical actuals: {str(ex)}")
            return

        if self.upload_s3:
            os.remove(file_name)

        self.stdout.write(
            self.style.SUCCESS(
                f"Uploaded historical actuals for year {year}. "))
Esempio n. 3
0
    def test_budget_file_contains_dash(self):
        self.assertEqual(
            BudgetMonthlyFigure.objects.filter(
                financial_code__cost_centre=self.cost_centre_code).count(),
            0,
        )

        actual_month = 4
        FinancialPeriod.objects.filter(
            financial_period_code=actual_month).update(actual_loaded=True)

        good_file_upload = FileUpload(
            s3_document_file=os.path.join(
                os.path.dirname(__file__),
                "test_assets/budget_upload_bad_dash.xlsx",
            ),
            uploading_user=self.test_user,
            document_type=FileUpload.BUDGET,
        )
        good_file_upload.save()

        upload_budget_from_file(
            good_file_upload,
            self.test_year,
        )

        self.assertEqual(
            BudgetMonthlyFigure.objects.filter(
                financial_year=self.test_year).count(),
            16,
        )
        # # Check that existing figures for the same period have been deleted
        self.assertEqual(
            BudgetMonthlyFigure.objects.filter(
                financial_year=self.test_year,
                financial_code__cost_centre=self.cost_centre_code,
            ).count(),
            8,
        )
        # Check that there are no entry for the actual periods
        for period in range(1, actual_month + 1):
            self.assertEqual(
                BudgetMonthlyFigure.objects.filter(
                    financial_year=self.test_year,
                    financial_code__cost_centre=self.cost_centre_code,
                    financial_period=period,
                ).first(),
                None,
            )
        self.assertEqual(
            BudgetMonthlyFigure.objects.filter(
                financial_year=self.test_year,
                financial_code__cost_centre=self.cost_centre_code,
                financial_period=12,
            ).first().amount,
            2200,
        )
Esempio n. 4
0
 def test_upload_wrong(self):
     file_upload_obj = FileUpload(
         document_file_name=self.excel_file_name,
         document_type=FileUpload.PREVIOUSYEAR,
         file_location=FileUpload.LOCALFILE,
     )
     file_upload_obj.save()
     with self.assertRaises(ArchiveYearError):
         upload_previous_year(
             self.data_worksheet,
             self.archived_year + 1,
             file_upload_obj,
         )
Esempio n. 5
0
    def setUp(self):
        self.client.force_login(self.test_user)
        self.test_year = 2019
        make_financial_year_current(self.test_year)
        self.test_period = 9

        self.cost_centre_code = TEST_COST_CENTRE
        self.valid_natural_account_code = TEST_VALID_NATURAL_ACCOUNT_CODE
        self.not_valid_natural_account_code = TEST_NOT_VALID_NATURAL_ACCOUNT_CODE
        self.programme_code = TEST_PROGRAMME_CODE
        self.test_amount = 100
        self.directorate_obj = DirectorateFactory.create(
            directorate_code='T123')
        CostCentreFactory.create(
            cost_centre_code=self.cost_centre_code,
            directorate=self.directorate_obj,
            active=False,
        )
        NaturalCodeFactory.create(
            natural_account_code=self.valid_natural_account_code,
            economic_budget_code=VALID_ECONOMIC_CODE_LIST[0],
            active=False,
        )
        NaturalCodeFactory.create(
            natural_account_code=18162001,
            economic_budget_code=VALID_ECONOMIC_CODE_LIST[0],
            active=False,
        )
        NaturalCodeFactory.create(
            natural_account_code=self.not_valid_natural_account_code,
            active=False,
        )
        ProgrammeCodeFactory.create(
            programme_code=self.programme_code,
            active=False,
        )
        ProgrammeCodeFactory.create(programme_code='310540')
        ProgrammeCodeFactory.create(programme_code='310530')

        self.period_obj = FinancialPeriod.objects.get(
            period_calendar_code=self.test_period)
        self.year_obj = FinancialYear.objects.get(financial_year=2019)
        dummy_upload = FileUpload(
            s3_document_file='dummy.csv',
            uploading_user=self.test_user,
            document_type=FileUpload.ACTUALS,
        )
        dummy_upload.save()
        self.check_financial_code = CheckFinancialCode(dummy_upload)
Esempio n. 6
0
    def test_budget_file_with_spaces_and_blanks(self):
        good_file_upload = FileUpload(
            s3_document_file=os.path.join(
                os.path.dirname(__file__),
                "test_assets/budget_upload_blank_data.xlsx",
            ),
            uploading_user=self.test_user,
            document_type=FileUpload.BUDGET,
        )
        good_file_upload.save()

        upload_budget_from_file(
            good_file_upload,
            self.test_year,
        )

        # # Check that existing figures for the same period have been deleted
        self.assertEqual(
            BudgetMonthlyFigure.objects.filter(
                financial_year=self.test_year).count(),
            24,
        )
        # # Check that existing figures for the same period have been deleted
        self.assertEqual(
            BudgetMonthlyFigure.objects.filter(
                financial_year=self.test_year,
                financial_code__cost_centre=self.cost_centre_code,
            ).count(),
            12,
        )
        # Check that figures for same budgets are added together
        self.assertEqual(
            BudgetMonthlyFigure.objects.filter(
                financial_year=self.test_year,
                financial_code__cost_centre=self.cost_centre_code,
                financial_period=1,
            ).first().amount,
            1100,
        )
        self.assertEqual(
            BudgetMonthlyFigure.objects.filter(
                financial_year=self.test_year,
                financial_code__cost_centre=self.cost_centre_code,
                financial_period=12,
            ).first().amount,
            2200,
        )
Esempio n. 7
0
    def handle(self, *args, **options):
        path = options["path"]
        month = options["month"]
        year = options["financial_year"]
        file_name = self.path_to_upload(path, 'xslx')

        fileobj = FileUpload(
            document_file_name=file_name,
            document_type=FileUpload.ACTUALS,
            file_location=FileUpload.LOCALFILE,
        )
        fileobj.save()
        upload_trial_balance_report(fileobj, month, year)
        if self.upload_s3:
            os.remove(file_name)

        self.stdout.write(
            self.style.SUCCESS("Actual for period {} added".format(month)))
Esempio n. 8
0
    def post(self, request, *args, **kwargs):
        form_class = self.get_form_class()
        form = self.get_form(form_class)

        logger.info("Received file upload attempt")

        if form.is_valid():
            logger.info("File upload form is valid")
            data = form.cleaned_data

            # When using a model form, you must use the
            # name attribute of the file rather than
            # passing the request file var directly as this is the
            # required when using the chunk uploader project
            s3_file_name = request.FILES['file'].name

            logger.info(f"s3_file_name is f{s3_file_name}")

            file_upload = FileUpload(
                s3_document_file=s3_file_name,
                uploading_user=request.user,
                document_type=FileUpload.ACTUALS,
            )
            file_upload.save()

            logger.info("Saved file to S3")

            # Process file async
            if settings.ASYNC_FILE_UPLOAD:
                logger.info("Using worker to upload file")
                process_uploaded_file.delay(
                    data['period'].period_calendar_code,
                    data['year'].financial_year,
                )
            else:
                process_uploaded_file(
                    data['period'].period_calendar_code,
                    data['year'].financial_year,
                )

            return self.form_valid(form)
        else:
            return self.form_invalid(form)
Esempio n. 9
0
    def test_upload(self):
        self.assertEqual(ArchivedFinancialCode.objects.all().count(), 0)
        self.assertEqual(ArchivedForecastData.objects.all().count(), 0)

        file_upload_obj = FileUpload(
            document_file_name=self.excel_file_name,
            document_type=FileUpload.PREVIOUSYEAR,
            file_location=FileUpload.LOCALFILE,
        )
        financial_year_obj = FinancialYear.objects.get(pk=self.archived_year)
        self.assertEqual(financial_year_obj.archived, False)
        file_upload_obj.save()
        upload_previous_year(
            self.data_worksheet,
            self.archived_year,
            file_upload_obj,
        )

        financial_year_obj = FinancialYear.objects.get(pk=self.archived_year)
        self.assertEqual(financial_year_obj.archived, True)

        self.assertEqual(ArchivedFinancialCode.objects.all().count(), 1)
        self.assertEqual(ArchivedForecastData.objects.all().count(), 1)
        result_obj = ArchivedForecastData.objects.all().first()
        self.assertEqual(self.results[0], result_obj.budget)
        self.assertEqual(self.results[1], result_obj.apr)
        self.assertEqual(self.results[2], result_obj.may)
        self.assertEqual(self.results[3], result_obj.jun)
        self.assertEqual(self.results[4], result_obj.jul)
        self.assertEqual(self.results[5], result_obj.aug)
        self.assertEqual(self.results[6], result_obj.sep)
        self.assertEqual(self.results[7], result_obj.oct)
        self.assertEqual(self.results[8], result_obj.nov)
        self.assertEqual(self.results[9], result_obj.dec)
        self.assertEqual(self.results[10], result_obj.jan)
        self.assertEqual(self.results[11], result_obj.feb)
        self.assertEqual(self.results[12], result_obj.mar)
        self.assertEqual(self.results[13], result_obj.adj1)
        self.assertEqual(self.results[14], result_obj.adj2)
        self.assertEqual(self.results[15], result_obj.adj3)
Esempio n. 10
0
 def setUp(self):
     super().setUp()
     dummy_upload = FileUpload(
         s3_document_file="dummy.csv",
         uploading_user=self.test_user,
         document_type=FileUpload.ACTUALS,
     )
     dummy_upload.save()
     self.check_financial_code = CheckArchivedFinancialCode(
         self.archived_year, dummy_upload)
     self.year_obj = FinancialYear.objects.get(
         financial_year=self.archived_year)
     self.year_obj.current = False
     self.year_obj.save()
     self.chart_of_account_line_correct = (f"3000-30000-"
                                           f"{self.cost_centre_code}-"
                                           f"{self.natural_account_code}-"
                                           f"{self.programme_code}-"
                                           f"{self.analisys1}-"
                                           f"{self.analisys2}-"
                                           f"{self.project_code}-"
                                           f"0000-"
                                           f"0000-0000")
Esempio n. 11
0
    def test_upload_trial_balance_report(self):
        # Check that BadZipFile is raised on
        # supply of incorrect file format
        bad_file_type_upload = FileUpload(
            s3_document_file=os.path.join(
                os.path.dirname(__file__),
                'test_assets/bad_file_type.csv',
            ),
            uploading_user=self.test_user,
            document_type=FileUpload.ACTUALS,
        )
        bad_file_type_upload.save()
        with self.assertRaises(BadZipFile):
            upload_trial_balance_report(
                bad_file_type_upload,
                self.test_period,
                self.test_year,
            )

        bad_title_file_upload = FileUpload(
            s3_document_file=os.path.join(
                os.path.dirname(__file__),
                'test_assets/bad_title_upload_test.xlsx',
            ),
            uploading_user=self.test_user,
            document_type=FileUpload.ACTUALS,
        )
        bad_title_file_upload.save()

        with self.assertRaises(UploadFileFormatError):
            upload_trial_balance_report(
                bad_title_file_upload,
                self.test_period,
                self.test_year,
            )

        self.assertEqual(
            FinancialCode.objects.filter(
                cost_centre=self.cost_centre_code).count(),
            0,
        )
        cost_centre_code_1 = 888888
        CostCentreFactory.create(cost_centre_code=cost_centre_code_1,
                                 directorate=self.directorate_obj)
        # Prepare to upload data. Create some data that will be deleted
        save_trial_balance_row(
            '3000-30000-{}-{}-{}-00000-00000-0000-0000-0000'.format(
                cost_centre_code_1, self.valid_natural_account_code,
                self.programme_code), self.test_amount, self.period_obj,
            self.year_obj, self.check_financial_code, 2)

        self.assertEqual(
            ForecastMonthlyFigure.objects.filter(
                financial_code__cost_centre=cost_centre_code_1, ).count(),
            0,
        )

        self.assertEqual(
            ActualUploadMonthlyFigure.objects.filter(
                financial_code__cost_centre=cost_centre_code_1, ).count(),
            1,
        )

        copy_current_year_actuals_to_monthly_figure(self.period_obj,
                                                    self.test_year)
        self.assertEqual(
            ForecastMonthlyFigure.objects.filter(
                financial_code__cost_centre=cost_centre_code_1, ).count(),
            1,
        )

        self.assertEqual(
            ActualUploadMonthlyFigure.objects.filter(
                financial_code__cost_centre=cost_centre_code_1, ).count(),
            0,
        )

        self.assertFalse(
            FinancialPeriod.objects.get(
                period_calendar_code=self.test_period).actual_loaded)
        bad_file_upload = FileUpload(
            s3_document_file=os.path.join(
                os.path.dirname(__file__),
                'test_assets/upload_bad_data.xlsx',
            ),
            uploading_user=self.test_user,
            document_type=FileUpload.ACTUALS,
        )
        bad_file_upload.save()

        upload_trial_balance_report(
            bad_file_upload,
            self.test_period,
            self.test_year,
        )

        self.assertFalse(
            FinancialPeriod.objects.get(
                period_calendar_code=self.test_period, ).actual_loaded)

        self.assertEqual(
            ForecastMonthlyFigure.objects.filter(
                financial_code__cost_centre=cost_centre_code_1).count(),
            1,
        )

        good_file_upload = FileUpload(
            s3_document_file=os.path.join(
                os.path.dirname(__file__),
                'test_assets/upload_test.xlsx',
            ),
            uploading_user=self.test_user,
            document_type=FileUpload.ACTUALS,
        )
        good_file_upload.save()

        upload_trial_balance_report(
            good_file_upload,
            self.test_period,
            self.test_year,
        )
        # Check that existing figures for the same period have been deleted
        self.assertEqual(
            ForecastMonthlyFigure.objects.filter(
                financial_code__cost_centre=cost_centre_code_1).count(),
            0,
        )
        # Check for existence of monthly figures
        self.assertEqual(
            ForecastMonthlyFigure.objects.filter(
                financial_code__cost_centre=self.cost_centre_code).count(),
            4,
        )
        result = ForecastMonthlyFigure.objects.filter(
            financial_code__cost_centre=self.cost_centre_code).aggregate(
                total=Sum('amount'))

        # Check that figures have correct values
        self.assertEqual(
            result['total'],
            1000000,
        )

        self.assertTrue(
            FinancialPeriod.objects.get(
                period_calendar_code=self.test_period).actual_loaded)
Esempio n. 12
0
def archive_current_year():
    # Get the latest period for archiving
    fields = ForecastQueryFields()
    financial_year = fields.selected_year

    try:
        validate_year_for_archiving_actuals(financial_year, False)
    except ArchiveYearError as ex:
        raise ex

    datamodel = fields.datamodel
    data_to_archive_list = datamodel.view_data.raw_data_annotated(
        fields.archive_forecast_columns, {}, year=financial_year)
    financial_year_obj = FinancialYear.objects.get(pk=financial_year)
    # Clear the table used to upload the previous_years.
    # The previous_years are uploaded to to a temporary storage,
    # and copied when the upload is completed successfully.
    # This means that we always have a full upload.
    ArchivedForecastDataUpload.objects.filter(
        financial_year=financial_year, ).delete()
    rows_to_process = data_to_archive_list.count()

    # Create an entry in the file upload table, even if it is not a file.
    # It is useful for keeping the log of errors
    file_upload = FileUpload(document_file_name="dummy",
                             document_type=FileUpload.PREVIOUSYEAR,
                             file_location=FileUpload.LOCALFILE,
                             status=FileUpload.PROCESSING)

    check_financial_code = CheckArchivedFinancialCode(financial_year,
                                                      file_upload)
    row_number = 0
    cost_centre_field = fields.cost_centre_code_field
    nac_field = fields.nac_code_field
    programme_field = fields.programme_code_field
    analysis1_field = fields.analysis1_code_field
    analysis2_field = fields.analysis2_code_field
    project_code_field = fields.project_code_field

    for row_to_archive in data_to_archive_list:
        row_number += 1
        if not row_number % 100:
            # Display the number of rows processed every 100 rows
            set_file_upload_feedback(
                file_upload,
                f"Processing row {row_number} of {rows_to_process}.")
            logger.info(f"Processing row {row_number} of {rows_to_process}.")

        cost_centre = row_to_archive[cost_centre_field]
        nac = row_to_archive[nac_field]
        programme_code = row_to_archive[programme_field]
        analysis1 = row_to_archive[analysis1_field]
        analysis2 = row_to_archive[analysis2_field]
        project_code = row_to_archive[project_code_field]

        check_financial_code.validate(
            cost_centre,
            nac,
            programme_code,
            analysis1,
            analysis2,
            project_code,
            row_number,
        )
        if not check_financial_code.error_found:
            financialcode_obj = check_financial_code.get_financial_code()
            try:
                archive_to_temp_previous_year_figures(
                    row_to_archive,
                    financial_year_obj,
                    financialcode_obj,
                )
            except (UploadFileFormatError, ArchiveYearError) as ex:
                set_file_upload_fatal_error(
                    file_upload,
                    str(ex),
                    str(ex),
                )
                raise ex

    final_status = set_final_status(check_financial_code)

    if final_status != FileUpload.PROCESSEDWITHERROR:
        # No errors, so we can copy the figures
        # from the temporary table to the previous_years
        copy_previous_year_figure_from_temp_table(financial_year)

    set_file_upload_feedback(file_upload, f"Processed {rows_to_process} rows.",
                             final_status)

    if final_status == FileUpload.PROCESSEDWITHERROR:
        raise UploadFileDataError(
            "No data archived. Check the log in the file upload record.")
Esempio n. 13
0
    def test_upload_budget_report(self):
        # Check that BadZipFile is raised on
        # supply of incorrect file format
        bad_file_type_upload = FileUpload(
            s3_document_file=os.path.join(
                os.path.dirname(__file__),
                "test_assets/bad_file_type.csv",
            ),
            uploading_user=self.test_user,
            document_type=FileUpload.BUDGET,
        )
        bad_file_type_upload.save()
        with self.assertRaises(BadZipFile):
            upload_budget_from_file(
                bad_file_type_upload,
                self.test_year,
            )

        bad_header_file_upload = FileUpload(
            s3_document_file=os.path.join(
                os.path.dirname(__file__),
                "test_assets/budget_upload_bad_header.xlsx",
            ),
            uploading_user=self.test_user,
            document_type=FileUpload.BUDGET,
        )
        bad_header_file_upload.save()

        with self.assertRaises(UploadFileFormatError):
            upload_budget_from_file(
                bad_header_file_upload,
                self.test_year,
            )
        # Check that the error is raised, and no data is uploaded
        bad_file_upload = FileUpload(
            s3_document_file=os.path.join(
                os.path.dirname(__file__),
                "test_assets/budget_upload_bad_data.xlsx",
            ),
            uploading_user=self.test_user,
            document_type=FileUpload.BUDGET,
        )
        bad_file_upload.save()

        self.assertEqual(
            BudgetMonthlyFigure.objects.all().count(),
            0,
        )
        upload_budget_from_file(
            bad_file_upload,
            self.test_year,
        )
        self.assertEqual(bad_file_upload.status, FileUpload.PROCESSEDWITHERROR)
        self.assertEqual(
            BudgetMonthlyFigure.objects.all().count(),
            0,
        )

        good_file_upload = FileUpload(
            s3_document_file=os.path.join(
                os.path.dirname(__file__),
                "test_assets/budget_upload_test.xlsx",
            ),
            uploading_user=self.test_user,
            document_type=FileUpload.BUDGET,
        )
        good_file_upload.save()

        upload_budget_from_file(
            good_file_upload,
            self.test_year,
        )

        # # Check that existing figures for the same period have been deleted
        self.assertEqual(
            BudgetMonthlyFigure.objects.filter(
                financial_year=self.test_year).count(),
            24,
        )
        # # Check that existing figures for the same period have been deleted
        self.assertEqual(
            BudgetMonthlyFigure.objects.filter(
                financial_year=self.test_year,
                financial_code__cost_centre=self.cost_centre_code,
            ).count(),
            12,
        )
        # Check that figures for same budgets are added together
        self.assertEqual(
            BudgetMonthlyFigure.objects.filter(
                financial_year=self.test_year,
                financial_code__cost_centre=self.cost_centre_code,
                financial_period=1,
            ).first().amount,
            1100,
        )
        self.assertEqual(
            BudgetMonthlyFigure.objects.filter(
                financial_year=self.test_year,
                financial_code__cost_centre=self.cost_centre_code,
                financial_period=12,
            ).first().amount,
            2200,
        )