def test_update_status_error_retries(self):
"""Test set_error() when MAX_RETRIES is exceeded."""
# set status to READY state.
self._setup_ready_status()
for idx in range(1, ProviderStatus.MAX_RETRIES + 2):
# log an error
with ProviderStatus(self.aws_test_provider_uuid) as accessor:
err = Exception(self.FAKE.word())
accessor.set_error(error=err)
# status should stay in WARNING until MAX_RETRIES is exceeded.
if idx < ProviderStatus.MAX_RETRIES:
with ProviderStatus(
self.aws_test_provider_uuid) as new_accessor:
self.assertEqual(new_accessor.get_status(),
ProviderStatusCode.WARNING)
self.assertEqual(new_accessor.get_retries(), idx)
# status should be DISABLED after MAX_RETRIES is reached.
with ProviderStatus(self.aws_test_provider_uuid) as other_accessor:
self.assertEqual(other_accessor.get_status(),
ProviderStatusCode.DISABLED_ERROR)
self.assertEqual(other_accessor.get_retries(),
ProviderStatus.MAX_RETRIES)
def _get_report_files(task, customer_name, authentication, billing_source,
provider_type, provider_uuid, report_month):
"""
Task to download a Report.
Args:
task (Object): Bound celery task.
customer_name (String): Name of the customer owning the cost usage report.
access_credential (String): Credential needed to access cost usage report
in the backend provider.
report_source (String): Location of the cost usage report in the backend provider.
provider_type (String): Koku defined provider type string. Example: Amazon = 'AWS'
provider_uuid (String): Provider uuid.
report_month (DateTime): Month for report to download.
Returns:
files (List) List of filenames with full local path.
Example: ['/var/tmp/masu/region/aws/catch-clearly.csv',
'/var/tmp/masu/base/aws/professor-hour-industry-television.csv']
"""
month_string = report_month.strftime("%B %Y")
log_statement = (f"Downloading report for:\n"
f" schema_name: {customer_name}\n"
f" provider: {provider_type}\n"
f" account (provider uuid): {provider_uuid}\n"
f" report_month: {month_string}")
LOG.info(log_statement)
try:
disk = psutil.disk_usage(Config.PVC_DIR)
disk_msg = f"Available disk space: {disk.free} bytes ({100 - disk.percent}%)"
except OSError:
disk_msg = f"Unable to find available disk space. {Config.PVC_DIR} does not exist"
LOG.info(disk_msg)
reports = None
try:
downloader = ReportDownloader(
task=task,
customer_name=customer_name,
access_credential=authentication,
report_source=billing_source,
provider_type=provider_type,
provider_uuid=provider_uuid,
report_name=None,
)
reports = downloader.download_report(report_month)
except (MasuProcessingError, MasuProviderError,
ReportDownloaderError) as err:
worker_stats.REPORT_FILE_DOWNLOAD_ERROR_COUNTER.labels(
provider_type=provider_type).inc()
LOG.error(str(err))
with ProviderStatus(provider_uuid) as status:
status.set_error(error=err)
raise err
with ProviderStatus(provider_uuid) as status:
status.set_status(ProviderStatusCode.READY)
return reports
def test_set_status_success(self):
"""Test set_status()."""
self._setup_random_status()
with ProviderStatus(self.aws_provider_uuid) as accessor:
accessor.set_status(ProviderStatusCode.READY)
with ProviderStatus(self.aws_provider_uuid) as new_accessor:
self.assertEqual(new_accessor.get_status(),
ProviderStatusCode.READY)
self.assertEqual(new_accessor.get_last_message(), 'none')
self.assertEqual(new_accessor.get_retries(), 0)
def test_is_valid_new(self):
"""Test when is_valid() should be False when status is NEW."""
status = {
'provider_id': self.provider_uuid,
'status': ProviderStatusCode.NEW,
'last_message': self.FAKE.word(),
'timestamp': DateAccessor().today(),
'retries': 3,
}
with ProviderStatus(self.aws_provider_uuid) as accessor:
accessor.add(**status)
with ProviderStatus(self.aws_provider_uuid) as accessor:
self.assertFalse(accessor.is_valid())
def test_is_valid_new(self):
"""Test when is_valid() should be False when status is NEW."""
status = {
"provider_id": self.provider_uuid,
"status": ProviderStatusCode.NEW,
"last_message": self.FAKE.word(),
"timestamp": DateAccessor().today(),
"retries": 3,
}
with ProviderStatus(self.aws_provider_uuid) as accessor:
accessor.add(**status)
with ProviderStatus(self.aws_provider_uuid) as accessor:
self.assertFalse(accessor.is_valid())
def test_is_valid_warn(self):
"""Test is_valid() should be True when status is WARNING."""
status = {
"provider_id": self.provider_uuid,
"status": ProviderStatusCode.WARNING,
"last_message": self.FAKE.word(),
"timestamp": DateAccessor().today(),
"retries": 3,
}
with ProviderStatus(self.aws_provider_uuid) as accessor:
accessor.add(**status)
accessor = ProviderStatus(self.aws_provider_uuid)
self.assertTrue(accessor.is_valid())
def test_is_valid_disabled(self):
"""Test when is_valid() should be False when status is DISABLED."""
status = {
'provider_id': self.provider_uuid,
'status': ProviderStatusCode.DISABLED_ERROR,
'last_message': self.FAKE.word(),
'timestamp': DateAccessor().today(),
'retries': 3,
}
with ProviderStatus(self.aws_provider_uuid) as accessor:
accessor.add(**status)
accessor = ProviderStatus(self.aws_provider_uuid)
self.assertFalse(accessor.is_valid())
def test_is_valid_warn(self):
"""Test is_valid() should be True when status is WARNING."""
status = {
'provider_id': self.provider_uuid,
'status': ProviderStatusCode.WARNING,
'last_message': self.FAKE.word(),
'timestamp': DateAccessor().today(),
'retries': 3,
}
with ProviderStatus(self.aws_provider_uuid) as accessor:
accessor.add(**status)
accessor = ProviderStatus(self.aws_provider_uuid)
self.assertTrue(accessor.is_valid())
def test_is_backing_off_true(self):
"""Test is_backing_off() is true within the appropriate time window."""
two_hours_ago = DateAccessor().today() - timedelta(hours=2)
status = {
'provider_id': self.provider_uuid,
'status': ProviderStatusCode.WARNING,
'last_message': self.FAKE.word(),
'timestamp': two_hours_ago,
'retries': 1,
}
with ProviderStatus(self.aws_provider_uuid) as accessor:
accessor.add(**status)
with ProviderStatus(self.aws_provider_uuid) as accessor:
self.assertTrue(accessor.is_backing_off())
def test_is_valid_ready(self):
"""Test is_valid() should be True when status is READY."""
self._setup_ready_status()
with ProviderStatus(self.aws_provider_uuid) as accessor:
self.assertTrue(accessor.is_valid())
accessor.close_session()
def test_set_error(self):
"""Test set_error()."""
# set status to READY state.
self._setup_ready_status()
# log an error
accessor = ProviderStatus(self.aws_provider_uuid)
err = Exception(self.FAKE.word())
accessor.set_error(error=err)
# test that state moved from READY to WARNING
with ProviderStatus(self.aws_provider_uuid) as new_accessor:
self.assertEqual(new_accessor.get_status(),
ProviderStatusCode.WARNING)
self.assertEqual(new_accessor.get_last_message(), str(err))
self.assertEqual(new_accessor.get_retries(), 1)
def test_is_backing_off_false(self):
"""Test is_backing_off() is false outside the appropriate time window."""
three_hours_ago = DateAccessor().today() - timedelta(hours=3)
status = {
"provider_id": self.provider_uuid,
"status": ProviderStatusCode.WARNING,
"last_message": self.FAKE.word(),
"timestamp": str(three_hours_ago),
"retries": 1,
}
with ProviderStatus(self.aws_provider_uuid) as accessor:
accessor.add(**status)
with ProviderStatus(self.aws_provider_uuid) as accessor:
self.assertFalse(accessor.is_backing_off())
def test_is_backing_off_false(self):
"""Test is_backing_off() is false outside the appropriate time window."""
three_hours_ago = DateAccessor().today() - timedelta(hours=3)
status = {
'provider_id': self.provider_uuid,
'status': ProviderStatusCode.WARNING,
'last_message': self.FAKE.word(),
'timestamp': str(three_hours_ago),
'retries': 1,
}
with ProviderStatus(self.aws_provider_uuid) as accessor:
accessor.add(**status)
with ProviderStatus(self.aws_provider_uuid) as accessor:
self.assertFalse(accessor.is_backing_off())
accessor.close_session()
def _setup_ready_status(self):
"""set status to READY state. """
ready_status = {
'provider_id': self.provider_uuid,
'status': ProviderStatusCode.READY,
'last_message': 'none',
'timestamp': DateAccessor().today(),
'retries': 0,
}
with ProviderStatus(self.aws_provider_uuid) as accessor:
accessor.add(**ready_status)
def _setup_ready_status(self):
"""Set status to READY state."""
ready_status = {
"provider_id": self.provider_uuid,
"status": ProviderStatusCode.READY,
"last_message": "none",
"timestamp": DateAccessor().today(),
"retries": 0,
}
with ProviderStatus(self.aws_provider_uuid) as accessor:
accessor.add(**ready_status)
def _setup_random_status(self):
"""Set up a randomized status for testing.
This is being done in a separate function instead of in setUp() to
facilitate testing the case where there is no status in the DB.
"""
self.test_status = {
'provider_id': self.provider_uuid,
'status': random.choice(list(ProviderStatusCode)),
'last_message': self.FAKE.word(),
'timestamp': DateAccessor().today(),
'retries': random.randint(0, 10),
}
with ProviderStatus(self.aws_provider_uuid) as accessor:
accessor.add(**self.test_status)
def prepare(self):
"""
Prepare a processing request for each account.
Scans the database for providers that have reports that need to be processed.
Any report it finds is queued to the appropriate celery task to download
and process those reports.
Args:
None
Returns:
(celery.result.AsyncResult) Async result for download request.
"""
async_result = None
for account in self._polling_accounts:
provider_uuid = account.get("provider_uuid")
report_months = self.get_reports(provider_uuid)
for month in report_months:
provider_status = ProviderStatus(provider_uuid)
if provider_status.is_valid(
) and not provider_status.is_backing_off():
LOG.info(
"Getting %s report files for account (provider uuid): %s",
month.strftime("%B %Y"),
provider_uuid,
)
account["report_month"] = month
async_result = (get_report_files.s(**account)
| summarize_reports.s()).apply_async()
LOG.info("Download queued - schema_name: %s, Task ID: %s",
account.get("schema_name"), str(async_result))
# update labels
labeler = AccountLabel(
auth=account.get("authentication"),
schema=account.get("schema_name"),
provider_type=account.get("provider_type"),
)
account_number, label = labeler.get_label_details()
if account_number:
LOG.info("Account: %s Label: %s updated.",
account_number, label)
else:
LOG.info(
"Provider skipped: %s Valid: %s Backing off: %s",
account.get("provider_uuid"),
provider_status.is_valid(),
provider_status.is_backing_off(),
)
return async_result
def prepare(self):
"""
Prepare a processing request for each account.
Args:
None
Returns:
(celery.result.AsyncResult) Async result for download request.
"""
async_result = None
for account in self._polling_accounts:
provider_uuid = account.get('provider_uuid')
report_months = self.get_reports(provider_uuid)
for month in report_months:
provider_status = ProviderStatus(provider_uuid)
if provider_status.is_valid(
) and not provider_status.is_backing_off():
LOG.info(
'Getting %s report files for account (provider uuid): %s',
month.strftime('%B %Y'), provider_uuid)
account['report_month'] = month
async_result = (get_report_files.s(**account) | summarize_reports.s()).\
apply_async()
LOG.info('Download queued - schema_name: %s, Task ID: %s',
account.get('schema_name'), str(async_result))
# update labels
labeler = AccountLabel(
auth=account.get('authentication'),
schema=account.get('schema_name'),
provider_type=account.get('provider_type'))
account_number, label = labeler.get_label_details()
if account_number:
LOG.info('Account: %s Label: %s updated.',
account_number, label)
else:
LOG.info('Provider skipped: %s Valid: %s Backing off: %s',
account.get('provider_uuid'),
provider_status.is_valid(),
provider_status.is_backing_off())
return async_result
def _get_report_files(customer_name,
authentication,
billing_source,
provider_type,
provider_uuid,
report_name=None):
"""
Task to download a Report.
Note that report_name will be not optional once Koku can specify
what report we should download.
Args:
customer_name (String): Name of the customer owning the cost usage report.
access_credential (String): Credential needed to access cost usage report
in the backend provider.
report_source (String): Location of the cost usage report in the backend provider.
provider_type (String): Koku defined provider type string. Example: Amazon = 'AWS'
provider_uuid (String): Provider uuid.
report_name (String): Name of the cost usage report to download.
Returns:
files (List) List of filenames with full local path.
Example: ['/var/tmp/masu/region/aws/catch-clearly.csv',
'/var/tmp/masu/base/aws/professor-hour-industry-television.csv']
"""
with ProviderDBAccessor(provider_uuid=provider_uuid) as provider_accessor:
reports_processed = provider_accessor.get_setup_complete()
provider_id = provider_accessor.get_provider().id
if Config.INGEST_OVERRIDE or not reports_processed:
number_of_months = Config.INITIAL_INGEST_NUM_MONTHS
else:
number_of_months = 2
stmt = ('Downloading report for'
' credential: {},'
' source: {},'
' customer_name: {},'
' provider: {},'
' number_of_months: {}')
log_statement = stmt.format(str(authentication), str(billing_source),
customer_name, provider_type, number_of_months)
LOG.info(log_statement)
try:
disk = psutil.disk_usage(Config.PVC_DIR)
disk_msg = 'Available disk space: {} bytes ({}%)'.format(
disk.free, 100 - disk.percent)
except OSError:
disk_msg = 'Unable to find available disk space. {} does not exist'.format(
Config.PVC_DIR)
LOG.info(disk_msg)
reports = None
try:
downloader = ReportDownloader(customer_name=customer_name,
access_credential=authentication,
report_source=billing_source,
provider_type=provider_type,
provider_id=provider_id,
report_name=report_name)
reports = downloader.get_reports(number_of_months)
except (MasuProcessingError, MasuProviderError,
ReportDownloaderError) as err:
worker_stats.REPORT_FILE_DOWNLOAD_ERROR_COUNTER.labels(
provider_type=provider_type).inc()
LOG.error(str(err))
with ProviderStatus(provider_uuid) as status:
status.set_error(error=err)
raise err
with ProviderStatus(provider_uuid) as status:
status.set_status(ProviderStatusCode.READY)
return reports
