def post(self, request, *args, **kwargs):
form_class = self.get_form_class()
form = self.get_form(form_class)
if form.is_valid():
data = form.cleaned_data
# When using a model form, you must use the
# name attribute of the file rather than
# passing the request file var directly as this is the
# required when using the chunk uploader project
s3_file_name = request.FILES['file'].name
file_upload = FileUpload(
s3_document_file=s3_file_name,
uploading_user=request.user,
document_type=FileUpload.BUDGET,
)
file_upload.save()
# Process file async
if settings.ASYNC_FILE_UPLOAD:
process_uploaded_file.delay(data['year'].financial_year, )
else:
process_uploaded_file(data['year'].financial_year, )
return self.form_valid(form)
else:
return self.form_invalid(form)
def handle(self, *args, **options):
path = options["path"]
year = options["financial_year"]
file_name = self.path_to_upload(path, 'xlsx')
fileobj = FileUpload(
document_file_name=file_name,
document_type=FileUpload.PREVIOUSYEAR,
file_location=FileUpload.LOCALFILE,
)
fileobj.save()
try:
upload_previous_year_from_file(fileobj, year)
except (WrongChartOFAccountCodeException, WrongArchivePeriodException,
UploadFileDataError, UploadFileFormatError,
ArchiveYearError) as ex:
raise CommandError(
f"Failure uploading historical actuals: {str(ex)}")
return
if self.upload_s3:
os.remove(file_name)
self.stdout.write(
self.style.SUCCESS(
f"Uploaded historical actuals for year {year}. "))
def test_budget_file_contains_dash(self):
self.assertEqual(
BudgetMonthlyFigure.objects.filter(
financial_code__cost_centre=self.cost_centre_code).count(),
0,
)
actual_month = 4
FinancialPeriod.objects.filter(
financial_period_code=actual_month).update(actual_loaded=True)
good_file_upload = FileUpload(
s3_document_file=os.path.join(
os.path.dirname(__file__),
"test_assets/budget_upload_bad_dash.xlsx",
),
uploading_user=self.test_user,
document_type=FileUpload.BUDGET,
)
good_file_upload.save()
upload_budget_from_file(
good_file_upload,
self.test_year,
)
self.assertEqual(
BudgetMonthlyFigure.objects.filter(
financial_year=self.test_year).count(),
16,
)
# # Check that existing figures for the same period have been deleted
self.assertEqual(
BudgetMonthlyFigure.objects.filter(
financial_year=self.test_year,
financial_code__cost_centre=self.cost_centre_code,
).count(),
8,
)
# Check that there are no entry for the actual periods
for period in range(1, actual_month + 1):
self.assertEqual(
BudgetMonthlyFigure.objects.filter(
financial_year=self.test_year,
financial_code__cost_centre=self.cost_centre_code,
financial_period=period,
).first(),
None,
)
self.assertEqual(
BudgetMonthlyFigure.objects.filter(
financial_year=self.test_year,
financial_code__cost_centre=self.cost_centre_code,
financial_period=12,
).first().amount,
2200,
)
def test_upload_wrong(self):
file_upload_obj = FileUpload(
document_file_name=self.excel_file_name,
document_type=FileUpload.PREVIOUSYEAR,
file_location=FileUpload.LOCALFILE,
)
file_upload_obj.save()
with self.assertRaises(ArchiveYearError):
upload_previous_year(
self.data_worksheet,
self.archived_year + 1,
file_upload_obj,
)
def setUp(self):
self.client.force_login(self.test_user)
self.test_year = 2019
make_financial_year_current(self.test_year)
self.test_period = 9
self.cost_centre_code = TEST_COST_CENTRE
self.valid_natural_account_code = TEST_VALID_NATURAL_ACCOUNT_CODE
self.not_valid_natural_account_code = TEST_NOT_VALID_NATURAL_ACCOUNT_CODE
self.programme_code = TEST_PROGRAMME_CODE
self.test_amount = 100
self.directorate_obj = DirectorateFactory.create(
directorate_code='T123')
CostCentreFactory.create(
cost_centre_code=self.cost_centre_code,
directorate=self.directorate_obj,
active=False,
)
NaturalCodeFactory.create(
natural_account_code=self.valid_natural_account_code,
economic_budget_code=VALID_ECONOMIC_CODE_LIST[0],
active=False,
)
NaturalCodeFactory.create(
natural_account_code=18162001,
economic_budget_code=VALID_ECONOMIC_CODE_LIST[0],
active=False,
)
NaturalCodeFactory.create(
natural_account_code=self.not_valid_natural_account_code,
active=False,
)
ProgrammeCodeFactory.create(
programme_code=self.programme_code,
active=False,
)
ProgrammeCodeFactory.create(programme_code='310540')
ProgrammeCodeFactory.create(programme_code='310530')
self.period_obj = FinancialPeriod.objects.get(
period_calendar_code=self.test_period)
self.year_obj = FinancialYear.objects.get(financial_year=2019)
dummy_upload = FileUpload(
s3_document_file='dummy.csv',
uploading_user=self.test_user,
document_type=FileUpload.ACTUALS,
)
dummy_upload.save()
self.check_financial_code = CheckFinancialCode(dummy_upload)
def test_budget_file_with_spaces_and_blanks(self):
good_file_upload = FileUpload(
s3_document_file=os.path.join(
os.path.dirname(__file__),
"test_assets/budget_upload_blank_data.xlsx",
),
uploading_user=self.test_user,
document_type=FileUpload.BUDGET,
)
good_file_upload.save()
upload_budget_from_file(
good_file_upload,
self.test_year,
)
# # Check that existing figures for the same period have been deleted
self.assertEqual(
BudgetMonthlyFigure.objects.filter(
financial_year=self.test_year).count(),
24,
)
# # Check that existing figures for the same period have been deleted
self.assertEqual(
BudgetMonthlyFigure.objects.filter(
financial_year=self.test_year,
financial_code__cost_centre=self.cost_centre_code,
).count(),
12,
)
# Check that figures for same budgets are added together
self.assertEqual(
BudgetMonthlyFigure.objects.filter(
financial_year=self.test_year,
financial_code__cost_centre=self.cost_centre_code,
financial_period=1,
).first().amount,
1100,
)
self.assertEqual(
BudgetMonthlyFigure.objects.filter(
financial_year=self.test_year,
financial_code__cost_centre=self.cost_centre_code,
financial_period=12,
).first().amount,
2200,
)
def handle(self, *args, **options):
path = options["path"]
month = options["month"]
year = options["financial_year"]
file_name = self.path_to_upload(path, 'xslx')
fileobj = FileUpload(
document_file_name=file_name,
document_type=FileUpload.ACTUALS,
file_location=FileUpload.LOCALFILE,
)
fileobj.save()
upload_trial_balance_report(fileobj, month, year)
if self.upload_s3:
os.remove(file_name)
self.stdout.write(
self.style.SUCCESS("Actual for period {} added".format(month)))
def post(self, request, *args, **kwargs):
form_class = self.get_form_class()
form = self.get_form(form_class)
logger.info("Received file upload attempt")
if form.is_valid():
logger.info("File upload form is valid")
data = form.cleaned_data
# When using a model form, you must use the
# name attribute of the file rather than
# passing the request file var directly as this is the
# required when using the chunk uploader project
s3_file_name = request.FILES['file'].name
logger.info(f"s3_file_name is f{s3_file_name}")
file_upload = FileUpload(
s3_document_file=s3_file_name,
uploading_user=request.user,
document_type=FileUpload.ACTUALS,
)
file_upload.save()
logger.info("Saved file to S3")
# Process file async
if settings.ASYNC_FILE_UPLOAD:
logger.info("Using worker to upload file")
process_uploaded_file.delay(
data['period'].period_calendar_code,
data['year'].financial_year,
)
else:
process_uploaded_file(
data['period'].period_calendar_code,
data['year'].financial_year,
)
return self.form_valid(form)
else:
return self.form_invalid(form)
def test_upload(self):
self.assertEqual(ArchivedFinancialCode.objects.all().count(), 0)
self.assertEqual(ArchivedForecastData.objects.all().count(), 0)
file_upload_obj = FileUpload(
document_file_name=self.excel_file_name,
document_type=FileUpload.PREVIOUSYEAR,
file_location=FileUpload.LOCALFILE,
)
financial_year_obj = FinancialYear.objects.get(pk=self.archived_year)
self.assertEqual(financial_year_obj.archived, False)
file_upload_obj.save()
upload_previous_year(
self.data_worksheet,
self.archived_year,
file_upload_obj,
)
financial_year_obj = FinancialYear.objects.get(pk=self.archived_year)
self.assertEqual(financial_year_obj.archived, True)
self.assertEqual(ArchivedFinancialCode.objects.all().count(), 1)
self.assertEqual(ArchivedForecastData.objects.all().count(), 1)
result_obj = ArchivedForecastData.objects.all().first()
self.assertEqual(self.results[0], result_obj.budget)
self.assertEqual(self.results[1], result_obj.apr)
self.assertEqual(self.results[2], result_obj.may)
self.assertEqual(self.results[3], result_obj.jun)
self.assertEqual(self.results[4], result_obj.jul)
self.assertEqual(self.results[5], result_obj.aug)
self.assertEqual(self.results[6], result_obj.sep)
self.assertEqual(self.results[7], result_obj.oct)
self.assertEqual(self.results[8], result_obj.nov)
self.assertEqual(self.results[9], result_obj.dec)
self.assertEqual(self.results[10], result_obj.jan)
self.assertEqual(self.results[11], result_obj.feb)
self.assertEqual(self.results[12], result_obj.mar)
self.assertEqual(self.results[13], result_obj.adj1)
self.assertEqual(self.results[14], result_obj.adj2)
self.assertEqual(self.results[15], result_obj.adj3)
def setUp(self):
super().setUp()
dummy_upload = FileUpload(
s3_document_file="dummy.csv",
uploading_user=self.test_user,
document_type=FileUpload.ACTUALS,
)
dummy_upload.save()
self.check_financial_code = CheckArchivedFinancialCode(
self.archived_year, dummy_upload)
self.year_obj = FinancialYear.objects.get(
financial_year=self.archived_year)
self.year_obj.current = False
self.year_obj.save()
self.chart_of_account_line_correct = (f"3000-30000-"
f"{self.cost_centre_code}-"
f"{self.natural_account_code}-"
f"{self.programme_code}-"
f"{self.analisys1}-"
f"{self.analisys2}-"
f"{self.project_code}-"
f"0000-"
f"0000-0000")
def test_upload_trial_balance_report(self):
# Check that BadZipFile is raised on
# supply of incorrect file format
bad_file_type_upload = FileUpload(
s3_document_file=os.path.join(
os.path.dirname(__file__),
'test_assets/bad_file_type.csv',
),
uploading_user=self.test_user,
document_type=FileUpload.ACTUALS,
)
bad_file_type_upload.save()
with self.assertRaises(BadZipFile):
upload_trial_balance_report(
bad_file_type_upload,
self.test_period,
self.test_year,
)
bad_title_file_upload = FileUpload(
s3_document_file=os.path.join(
os.path.dirname(__file__),
'test_assets/bad_title_upload_test.xlsx',
),
uploading_user=self.test_user,
document_type=FileUpload.ACTUALS,
)
bad_title_file_upload.save()
with self.assertRaises(UploadFileFormatError):
upload_trial_balance_report(
bad_title_file_upload,
self.test_period,
self.test_year,
)
self.assertEqual(
FinancialCode.objects.filter(
cost_centre=self.cost_centre_code).count(),
0,
)
cost_centre_code_1 = 888888
CostCentreFactory.create(cost_centre_code=cost_centre_code_1,
directorate=self.directorate_obj)
# Prepare to upload data. Create some data that will be deleted
save_trial_balance_row(
'3000-30000-{}-{}-{}-00000-00000-0000-0000-0000'.format(
cost_centre_code_1, self.valid_natural_account_code,
self.programme_code), self.test_amount, self.period_obj,
self.year_obj, self.check_financial_code, 2)
self.assertEqual(
ForecastMonthlyFigure.objects.filter(
financial_code__cost_centre=cost_centre_code_1, ).count(),
0,
)
self.assertEqual(
ActualUploadMonthlyFigure.objects.filter(
financial_code__cost_centre=cost_centre_code_1, ).count(),
1,
)
copy_current_year_actuals_to_monthly_figure(self.period_obj,
self.test_year)
self.assertEqual(
ForecastMonthlyFigure.objects.filter(
financial_code__cost_centre=cost_centre_code_1, ).count(),
1,
)
self.assertEqual(
ActualUploadMonthlyFigure.objects.filter(
financial_code__cost_centre=cost_centre_code_1, ).count(),
0,
)
self.assertFalse(
FinancialPeriod.objects.get(
period_calendar_code=self.test_period).actual_loaded)
bad_file_upload = FileUpload(
s3_document_file=os.path.join(
os.path.dirname(__file__),
'test_assets/upload_bad_data.xlsx',
),
uploading_user=self.test_user,
document_type=FileUpload.ACTUALS,
)
bad_file_upload.save()
upload_trial_balance_report(
bad_file_upload,
self.test_period,
self.test_year,
)
self.assertFalse(
FinancialPeriod.objects.get(
period_calendar_code=self.test_period, ).actual_loaded)
self.assertEqual(
ForecastMonthlyFigure.objects.filter(
financial_code__cost_centre=cost_centre_code_1).count(),
1,
)
good_file_upload = FileUpload(
s3_document_file=os.path.join(
os.path.dirname(__file__),
'test_assets/upload_test.xlsx',
),
uploading_user=self.test_user,
document_type=FileUpload.ACTUALS,
)
good_file_upload.save()
upload_trial_balance_report(
good_file_upload,
self.test_period,
self.test_year,
)
# Check that existing figures for the same period have been deleted
self.assertEqual(
ForecastMonthlyFigure.objects.filter(
financial_code__cost_centre=cost_centre_code_1).count(),
0,
)
# Check for existence of monthly figures
self.assertEqual(
ForecastMonthlyFigure.objects.filter(
financial_code__cost_centre=self.cost_centre_code).count(),
4,
)
result = ForecastMonthlyFigure.objects.filter(
financial_code__cost_centre=self.cost_centre_code).aggregate(
total=Sum('amount'))
# Check that figures have correct values
self.assertEqual(
result['total'],
1000000,
)
self.assertTrue(
FinancialPeriod.objects.get(
period_calendar_code=self.test_period).actual_loaded)
def archive_current_year():
# Get the latest period for archiving
fields = ForecastQueryFields()
financial_year = fields.selected_year
try:
validate_year_for_archiving_actuals(financial_year, False)
except ArchiveYearError as ex:
raise ex
datamodel = fields.datamodel
data_to_archive_list = datamodel.view_data.raw_data_annotated(
fields.archive_forecast_columns, {}, year=financial_year)
financial_year_obj = FinancialYear.objects.get(pk=financial_year)
# Clear the table used to upload the previous_years.
# The previous_years are uploaded to to a temporary storage,
# and copied when the upload is completed successfully.
# This means that we always have a full upload.
ArchivedForecastDataUpload.objects.filter(
financial_year=financial_year, ).delete()
rows_to_process = data_to_archive_list.count()
# Create an entry in the file upload table, even if it is not a file.
# It is useful for keeping the log of errors
file_upload = FileUpload(document_file_name="dummy",
document_type=FileUpload.PREVIOUSYEAR,
file_location=FileUpload.LOCALFILE,
status=FileUpload.PROCESSING)
check_financial_code = CheckArchivedFinancialCode(financial_year,
file_upload)
row_number = 0
cost_centre_field = fields.cost_centre_code_field
nac_field = fields.nac_code_field
programme_field = fields.programme_code_field
analysis1_field = fields.analysis1_code_field
analysis2_field = fields.analysis2_code_field
project_code_field = fields.project_code_field
for row_to_archive in data_to_archive_list:
row_number += 1
if not row_number % 100:
# Display the number of rows processed every 100 rows
set_file_upload_feedback(
file_upload,
f"Processing row {row_number} of {rows_to_process}.")
logger.info(f"Processing row {row_number} of {rows_to_process}.")
cost_centre = row_to_archive[cost_centre_field]
nac = row_to_archive[nac_field]
programme_code = row_to_archive[programme_field]
analysis1 = row_to_archive[analysis1_field]
analysis2 = row_to_archive[analysis2_field]
project_code = row_to_archive[project_code_field]
check_financial_code.validate(
cost_centre,
nac,
programme_code,
analysis1,
analysis2,
project_code,
row_number,
)
if not check_financial_code.error_found:
financialcode_obj = check_financial_code.get_financial_code()
try:
archive_to_temp_previous_year_figures(
row_to_archive,
financial_year_obj,
financialcode_obj,
)
except (UploadFileFormatError, ArchiveYearError) as ex:
set_file_upload_fatal_error(
file_upload,
str(ex),
str(ex),
)
raise ex
final_status = set_final_status(check_financial_code)
if final_status != FileUpload.PROCESSEDWITHERROR:
# No errors, so we can copy the figures
# from the temporary table to the previous_years
copy_previous_year_figure_from_temp_table(financial_year)
set_file_upload_feedback(file_upload, f"Processed {rows_to_process} rows.",
final_status)
if final_status == FileUpload.PROCESSEDWITHERROR:
raise UploadFileDataError(
"No data archived. Check the log in the file upload record.")
def test_upload_budget_report(self):
# Check that BadZipFile is raised on
# supply of incorrect file format
bad_file_type_upload = FileUpload(
s3_document_file=os.path.join(
os.path.dirname(__file__),
"test_assets/bad_file_type.csv",
),
uploading_user=self.test_user,
document_type=FileUpload.BUDGET,
)
bad_file_type_upload.save()
with self.assertRaises(BadZipFile):
upload_budget_from_file(
bad_file_type_upload,
self.test_year,
)
bad_header_file_upload = FileUpload(
s3_document_file=os.path.join(
os.path.dirname(__file__),
"test_assets/budget_upload_bad_header.xlsx",
),
uploading_user=self.test_user,
document_type=FileUpload.BUDGET,
)
bad_header_file_upload.save()
with self.assertRaises(UploadFileFormatError):
upload_budget_from_file(
bad_header_file_upload,
self.test_year,
)
# Check that the error is raised, and no data is uploaded
bad_file_upload = FileUpload(
s3_document_file=os.path.join(
os.path.dirname(__file__),
"test_assets/budget_upload_bad_data.xlsx",
),
uploading_user=self.test_user,
document_type=FileUpload.BUDGET,
)
bad_file_upload.save()
self.assertEqual(
BudgetMonthlyFigure.objects.all().count(),
0,
)
upload_budget_from_file(
bad_file_upload,
self.test_year,
)
self.assertEqual(bad_file_upload.status, FileUpload.PROCESSEDWITHERROR)
self.assertEqual(
BudgetMonthlyFigure.objects.all().count(),
0,
)
good_file_upload = FileUpload(
s3_document_file=os.path.join(
os.path.dirname(__file__),
"test_assets/budget_upload_test.xlsx",
),
uploading_user=self.test_user,
document_type=FileUpload.BUDGET,
)
good_file_upload.save()
upload_budget_from_file(
good_file_upload,
self.test_year,
)
# # Check that existing figures for the same period have been deleted
self.assertEqual(
BudgetMonthlyFigure.objects.filter(
financial_year=self.test_year).count(),
24,
)
# # Check that existing figures for the same period have been deleted
self.assertEqual(
BudgetMonthlyFigure.objects.filter(
financial_year=self.test_year,
financial_code__cost_centre=self.cost_centre_code,
).count(),
12,
)
# Check that figures for same budgets are added together
self.assertEqual(
BudgetMonthlyFigure.objects.filter(
financial_year=self.test_year,
financial_code__cost_centre=self.cost_centre_code,
financial_period=1,
).first().amount,
1100,
)
self.assertEqual(
BudgetMonthlyFigure.objects.filter(
financial_year=self.test_year,
financial_code__cost_centre=self.cost_centre_code,
financial_period=12,
).first().amount,
2200,
)