def test_task_is_running_false(self):
"""Test that a task is not running."""
task_list = [1, 2, 3]
_cache = WorkerCache()
_cache.set_host_specific_task_list(task_list)
self.assertFalse(_cache.task_is_running(4))
def __init__(self, task, download_path=None, **kwargs):
"""
Create a downloader.
Args:
task (Object) bound celery object
download_path (String) filesystem path to store downloaded files
Kwargs:
customer_name (String) customer name
access_credential (Dict) provider access credentials
report_source (String) cost report source
provider_type (String) cloud provider type
provider_uuid (String) cloud provider uuid
report_name (String) cost report name
"""
self._task = task
if download_path:
self.download_path = download_path
else:
self.download_path = mkdtemp(prefix="masu")
self.worker_cache = WorkerCache()
self._cache_key = kwargs.get("cache_key")
self._provider_uuid = kwargs.get("provider_uuid")
self.request_id = kwargs.get("request_id")
self.account = kwargs.get("account")
self.context = {"request_id": self.request_id, "provider_uuid": self._provider_uuid, "account": self.account}
def test_task_is_running_false(self):
"""Test that a task is not running."""
task_list = [1, 2, 3]
_cache = WorkerCache()
for task in task_list:
_cache.add_task_to_cache(task)
self.assertFalse(_cache.task_is_running(4))
def test_set_host_specific_task_list(self):
"""Test that setting a task list works."""
task_list = [1, 2, 3]
_cache = WorkerCache()
self.assertEqual(_cache.host_specific_worker_cache, [])
_cache.set_host_specific_task_list(task_list)
self.assertEqual(_cache.host_specific_worker_cache, task_list)
def test_add_task_to_cache(self):
"""Test that a single task is added."""
task_key = "task_key"
_cache = WorkerCache()
self.assertEqual(_cache.worker_cache, [])
_cache.add_task_to_cache(task_key)
self.assertEqual(_cache.worker_cache, [task_key])
def test_check_if_manifest_should_be_downloaded_task_currently_running(
self):
"""Test that a manifest being processed should not be reprocessed."""
_cache = WorkerCache()
_cache.add_task_to_cache(self.cache_key)
result = self.downloader.check_if_manifest_should_be_downloaded(
self.assembly_id)
self.assertFalse(result)
def test_get_all_running_tasks(self, mock_inspect):
"""Test that multiple hosts' task lists are combined."""
second_host = "koku-worker-2-sdfsdff"
first_host_list = [1, 2, 3]
second_host_list = [4, 5, 6]
expected = first_host_list + second_host_list
mock_worker_list = {
"celery@kokuworker": "",
f"celery@{second_host}": ""
}
mock_inspect.reserved.return_value = mock_worker_list
_cache = WorkerCache()
for task in first_host_list:
_cache.add_task_to_cache(task)
with override_settings(HOSTNAME=second_host):
_cache = WorkerCache()
for task in second_host_list:
_cache.add_task_to_cache(task)
self.assertEqual(sorted(_cache.get_all_running_tasks()),
sorted(expected))
def __init__(self, billing_source=None, provider_uuid=None):
"""
Orchestrator for processing.
Args:
billing_source (String): Individual account to retrieve.
"""
self._accounts, self._polling_accounts = self.get_accounts(
billing_source, provider_uuid)
self.worker_cache = WorkerCache()
def test_task_is_running_true(self, mock_inspect):
"""Test that a task is running."""
mock_worker_list = {"celery@kokuworker": ""}
mock_inspect.reserved.return_value = mock_worker_list
task_list = [1, 2, 3]
_cache = WorkerCache()
for task in task_list:
_cache.add_task_to_cache(task)
self.assertTrue(_cache.task_is_running(1))
def refresh_materialized_views( # noqa: C901
schema_name,
provider_type,
manifest_id=None,
provider_uuid="",
synchronous=False,
queue_name=None,
tracing_id=None,
):
"""Refresh the database's materialized views for reporting."""
task_name = "masu.processor.tasks.refresh_materialized_views"
cache_args = [schema_name, provider_type, provider_uuid]
if not synchronous:
worker_cache = WorkerCache()
if worker_cache.single_task_is_running(task_name, cache_args):
msg = f"Task {task_name} already running for {cache_args}. Requeuing."
LOG.info(log_json(tracing_id, msg))
refresh_materialized_views.s(
schema_name,
provider_type,
manifest_id=manifest_id,
provider_uuid=provider_uuid,
synchronous=synchronous,
queue_name=queue_name,
tracing_id=tracing_id,
).apply_async(queue=queue_name or REFRESH_MATERIALIZED_VIEWS_QUEUE)
return
worker_cache.lock_single_task(task_name,
cache_args,
timeout=settings.WORKER_CACHE_TIMEOUT)
materialized_views = ()
try:
with schema_context(schema_name):
for view in materialized_views:
table_name = view._meta.db_table
with connection.cursor() as cursor:
cursor.execute(
f"REFRESH MATERIALIZED VIEW CONCURRENTLY {table_name}")
LOG.info(log_json(tracing_id, f"Refreshed {table_name}."))
invalidate_view_cache_for_tenant_and_source_type(
schema_name, provider_type)
if provider_uuid:
ProviderDBAccessor(provider_uuid).set_data_updated_timestamp()
if manifest_id:
# Processing for this monifest should be complete after this step
with ReportManifestDBAccessor() as manifest_accessor:
manifest = manifest_accessor.get_manifest_by_id(manifest_id)
manifest_accessor.mark_manifest_as_completed(manifest)
except Exception as ex:
if not synchronous:
worker_cache.release_single_task(task_name, cache_args)
raise ex
if not synchronous:
worker_cache.release_single_task(task_name, cache_args)
def update_cost_model_costs(schema_name,
provider_uuid,
start_date=None,
end_date=None,
queue_name=None,
synchronous=False):
"""Update usage charge information.
Args:
schema_name (str) The DB schema name.
provider_uuid (str) The provider uuid.
start_date (str, Optional) - Start date of range to update derived cost.
end_date (str, Optional) - End date of range to update derived cost.
Returns
None
"""
task_name = "masu.processor.tasks.update_cost_model_costs"
cache_args = [schema_name, provider_uuid, start_date, end_date]
if not synchronous:
worker_cache = WorkerCache()
if worker_cache.single_task_is_running(task_name, cache_args):
msg = f"Task {task_name} already running for {cache_args}. Requeuing."
LOG.info(msg)
update_cost_model_costs.s(
schema_name,
provider_uuid,
start_date=start_date,
end_date=end_date,
queue_name=queue_name,
synchronous=synchronous,
).apply_async(queue=queue_name or UPDATE_COST_MODEL_COSTS_QUEUE)
return
worker_cache.lock_single_task(task_name, cache_args, timeout=600)
worker_stats.COST_MODEL_COST_UPDATE_ATTEMPTS_COUNTER.inc()
stmt = (f"update_cost_model_costs called with args:\n"
f" schema_name: {schema_name},\n"
f" provider_uuid: {provider_uuid}")
LOG.info(stmt)
try:
updater = CostModelCostUpdater(schema_name, provider_uuid)
if updater:
updater.update_cost_model_costs(start_date, end_date)
except Exception as ex:
if not synchronous:
worker_cache.release_single_task(task_name, cache_args)
raise ex
if not synchronous:
worker_cache.release_single_task(task_name, cache_args)
def test_get_all_running_tasks(self):
"""Test that multiple hosts' task lists are combined."""
second_host = "test"
first_host_list = [1, 2, 3]
second_host_list = [4, 5, 6]
expected = first_host_list + second_host_list
_cache = WorkerCache()
_cache.set_host_specific_task_list(first_host_list)
_worker_cache = _cache.worker_cache
_worker_cache[second_host] = [4, 5, 6]
cache.set(settings.WORKER_CACHE_KEY, _worker_cache, timeout=None)
self.assertEqual(_cache.get_all_running_tasks(), expected)
def test_active_worker_property(self, mock_inspect):
"""Test the active_workers property."""
test_matrix = [
{
"hostname": "celery@kokuworker",
"expected_workers": ["kokuworker"]
},
{
"hostname": "kokuworker",
"expected_workers": ["kokuworker"]
},
{
"hostname": "kokuworker&63)",
"expected_workers": ["kokuworker&63)"]
},
{
"hostname": "koku@worker&63)",
"expected_workers": ["worker&63)"]
},
{
"hostname": "",
"expected_workers": [""]
},
]
for test in test_matrix:
with self.subTest(test=test):
mock_worker_list = {test.get("hostname"): ""}
mock_inspect.reserved.return_value = mock_worker_list
_cache = WorkerCache()
self.assertEqual(_cache.active_workers,
test.get("expected_workers"))
def clear_worker_cache_on_shutdown(sender, **kwargs): # pragma: no cover
from masu.processor.worker_cache import WorkerCache
LOGGER.info("Clearing worker task cache.")
try:
WorkerCache().invalidate_host()
except Exception:
LOGGER.info("Cache not cleared on shutdown.")
def clear_worker_cache(sender, instance, **kwargs): # pragma: no cover
"""Clear WorkerCache after worker is up and running."""
from .database import check_migrations
from masu.processor.worker_cache import WorkerCache
while not check_migrations():
LOGGER.warning("Migrations not done. Sleeping")
time.sleep(5)
LOGGER.info("Clearing worker task cache.")
WorkerCache().invalidate_host()
def test_get_all_running_tasks(self):
"""Test that multiple hosts' task lists are combined."""
second_host = "test"
first_host_list = [1, 2, 3]
second_host_list = [4, 5, 6]
expected = first_host_list + second_host_list
_cache = WorkerCache()
for task in first_host_list:
_cache.add_task_to_cache(task)
with patch.object(settings, "HOSTNAME", second_host):
_cache = WorkerCache()
for task in second_host_list:
_cache.add_task_to_cache(task)
self.assertEqual(sorted(_cache.get_all_running_tasks()), sorted(expected))
def test_single_task_caching(self, mock_inspect):
"""Test that single task cache creates and deletes a cache entry."""
cache = WorkerCache()
task_name = "test_task"
task_args = ["schema1", "OCP"]
self.assertFalse(cache.single_task_is_running(task_name, task_args))
cache.lock_single_task(task_name, task_args)
self.assertTrue(cache.single_task_is_running(task_name, task_args))
cache.release_single_task(task_name, task_args)
self.assertFalse(cache.single_task_is_running(task_name, task_args))
def test_remove_task_from_cache(self):
"""Test that a task is removed."""
task_key = "task_key"
_cache = WorkerCache()
_cache.add_task_to_cache(task_key)
self.assertEqual(_cache.worker_cache, [task_key])
_cache.remove_task_from_cache(task_key)
self.assertEqual(_cache.worker_cache, [])
def test_remove_task_from_cache_value_not_in_cache(self):
"""Test that a task is removed."""
task_list = [1, 2, 3, 4]
_cache = WorkerCache()
_cache.set_host_specific_task_list(task_list)
self.assertEqual(_cache.host_specific_worker_cache, task_list)
_cache.remove_task_from_cache(5)
self.assertEqual(_cache.host_specific_worker_cache, task_list)
def refresh_materialized_views(schema_name,
provider_type,
manifest_id=None,
provider_uuid=None,
synchronous=False):
"""Refresh the database's materialized views for reporting."""
task_name = "masu.processor.tasks.refresh_materialized_views"
cache_args = [schema_name]
if not synchronous:
worker_cache = WorkerCache()
while worker_cache.single_task_is_running(task_name, cache_args):
time.sleep(5)
worker_cache.lock_single_task(task_name, cache_args)
materialized_views = ()
if provider_type in (Provider.PROVIDER_AWS, Provider.PROVIDER_AWS_LOCAL):
materialized_views = (AWS_MATERIALIZED_VIEWS +
OCP_ON_AWS_MATERIALIZED_VIEWS +
OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS)
elif provider_type in (Provider.PROVIDER_OCP):
materialized_views = (OCP_MATERIALIZED_VIEWS +
OCP_ON_AWS_MATERIALIZED_VIEWS +
OCP_ON_AZURE_MATERIALIZED_VIEWS +
OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS)
elif provider_type in (Provider.PROVIDER_AZURE,
Provider.PROVIDER_AZURE_LOCAL):
materialized_views = (AZURE_MATERIALIZED_VIEWS +
OCP_ON_AZURE_MATERIALIZED_VIEWS +
OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS)
with schema_context(schema_name):
for view in materialized_views:
table_name = view._meta.db_table
with connection.cursor() as cursor:
cursor.execute(
f"REFRESH MATERIALIZED VIEW CONCURRENTLY {table_name}")
LOG.info(f"Refreshed {table_name}.")
invalidate_view_cache_for_tenant_and_source_type(schema_name,
provider_type)
if provider_uuid:
ProviderDBAccessor(provider_uuid).set_data_updated_timestamp()
if manifest_id:
# Processing for this monifest should be complete after this step
with ReportManifestDBAccessor() as manifest_accessor:
manifest = manifest_accessor.get_manifest_by_id(manifest_id)
manifest_accessor.mark_manifest_as_completed(manifest)
if not synchronous:
worker_cache.release_single_task(task_name, cache_args)
def test_add_task_to_cache(self):
"""Test that a single task is added."""
task_list = [1, 2, 3]
expected = [1, 2, 3, 4]
_cache = WorkerCache()
_cache.set_host_specific_task_list(task_list)
self.assertEqual(_cache.host_specific_worker_cache, task_list)
_cache.add_task_to_cache(4)
self.assertEqual(_cache.host_specific_worker_cache, expected)
def test_remove_task_from_cache(self):
"""Test that a task is removed."""
task_list = [1, 2, 3, 4]
expected = [1, 2, 3]
_cache = WorkerCache()
_cache.set_host_specific_task_list(task_list)
self.assertEqual(_cache.host_specific_worker_cache, task_list)
_cache.remove_task_from_cache(4)
self.assertEqual(_cache.host_specific_worker_cache, expected)
def test_remove_task_from_cache_value_not_in_cache(self):
"""Test that a task is removed."""
task_list = [1, 2, 3, 4]
_cache = WorkerCache()
for task in task_list:
_cache.add_task_to_cache(task)
self.assertEqual(_cache.worker_cache, task_list)
_cache.remove_task_from_cache(5)
self.assertEqual(_cache.worker_cache, task_list)
def test_invalidate_host(self):
"""Test that a host's cache is invalidated."""
task_list = [1, 2, 3]
_cache = WorkerCache()
_cache.set_host_specific_task_list(task_list)
self.assertEqual(_cache.host_specific_worker_cache, task_list)
_cache.invalidate_host()
self.assertEqual(_cache.host_specific_worker_cache, [])
def test_invalidate_host(self):
"""Test that a host's cache is invalidated."""
task_list = [1, 2, 3]
_cache = WorkerCache()
for task in task_list:
_cache.add_task_to_cache(task)
self.assertEqual(_cache.worker_cache, task_list)
_cache.invalidate_host()
self.assertEqual(_cache.worker_cache, [])
def update_cost_model_costs(schema_name,
provider_uuid,
start_date=None,
end_date=None,
provider_type=None,
synchronous=False):
"""Update usage charge information.
Args:
schema_name (str) The DB schema name.
provider_uuid (str) The provider uuid.
start_date (str, Optional) - Start date of range to update derived cost.
end_date (str, Optional) - End date of range to update derived cost.
Returns
None
"""
task_name = "masu.processor.tasks.update_cost_model_costs"
cache_args = [schema_name, provider_uuid, start_date, end_date]
if not synchronous:
worker_cache = WorkerCache()
while worker_cache.single_task_is_running(task_name, cache_args):
time.sleep(5)
worker_cache.lock_single_task(task_name, cache_args, timeout=300)
worker_stats.COST_MODEL_COST_UPDATE_ATTEMPTS_COUNTER.inc()
stmt = (f"update_cost_model_costs called with args:\n"
f" schema_name: {schema_name},\n"
f" provider_uuid: {provider_uuid}")
LOG.info(stmt)
updater = CostModelCostUpdater(schema_name, provider_uuid)
if updater:
updater.update_cost_model_costs(start_date, end_date)
if not synchronous:
worker_cache.release_single_task(task_name, cache_args)
class ReportDownloaderBase:
"""
Download cost reports from a provider.
Base object class for downloading cost reports from a cloud provider.
"""
def __init__(self, task, download_path=None, **kwargs):
"""
Create a downloader.
Args:
task (Object) bound celery object
download_path (String) filesystem path to store downloaded files
Kwargs:
customer_name (String) customer name
access_credential (Dict) provider access credentials
report_source (String) cost report source
provider_type (String) cloud provider type
provider_uuid (String) cloud provider uuid
report_name (String) cost report name
"""
self._task = task
if download_path:
self.download_path = download_path
else:
self.download_path = mkdtemp(prefix="masu")
self.worker_cache = WorkerCache()
self._cache_key = kwargs.get("cache_key")
self._provider_uuid = kwargs.get("provider_uuid")
self.request_id = kwargs.get("request_id")
self.account = kwargs.get("account")
self.context = {"request_id": self.request_id, "provider_uuid": self._provider_uuid, "account": self.account}
def _get_existing_manifest_db_id(self, assembly_id):
"""Return a manifest DB object if it exists."""
manifest_id = None
with ReportManifestDBAccessor() as manifest_accessor:
manifest = manifest_accessor.get_manifest(assembly_id, self._provider_uuid)
if manifest:
manifest_id = manifest.id
return manifest_id
def check_if_manifest_should_be_downloaded(self, assembly_id):
"""Check if we should download this manifest.
We first check if we have a database record of this manifest.
That would indicate that we have already downloaded and at least
begun processing. We then check the last completed time for
a file in this manifest. This second check is to cover the case
when we did not complete processing and need to re-downlaod and
process the manifest.
Returns True if the manifest should be downloaded and processed.
"""
if self._cache_key and self.worker_cache.task_is_running(self._cache_key):
msg = f"{self._cache_key} is currently running."
LOG.info(log_json(self.request_id, msg, self.context))
return False
with ReportManifestDBAccessor() as manifest_accessor:
manifest = manifest_accessor.get_manifest(assembly_id, self._provider_uuid)
if manifest:
manifest_id = manifest.id
# check if `last_completed_datetime` is null for any report in the manifest.
# if nulls exist, report processing is not complete and reports should be downloaded.
need_to_download = manifest_accessor.is_last_completed_datetime_null(manifest_id)
if need_to_download:
self.worker_cache.add_task_to_cache(self._cache_key)
return need_to_download
# The manifest does not exist, this is the first time we are
# downloading and processing it.
self.worker_cache.add_task_to_cache(self._cache_key)
return True
def _process_manifest_db_record(self, assembly_id, billing_start, num_of_files):
"""Insert or update the manifest DB record."""
LOG.info("Inserting/updating manifest in database for assembly_id: %s", assembly_id)
with ReportManifestDBAccessor() as manifest_accessor:
manifest_entry = manifest_accessor.get_manifest(assembly_id, self._provider_uuid)
if not manifest_entry:
msg = f"No manifest entry found in database. Adding for bill period start: {billing_start}"
LOG.info(log_json(self.request_id, msg, self.context))
manifest_dict = {
"assembly_id": assembly_id,
"billing_period_start_datetime": billing_start,
"num_total_files": num_of_files,
"provider_uuid": self._provider_uuid,
"task": self._task.request.id,
}
manifest_entry = manifest_accessor.add(**manifest_dict)
manifest_accessor.mark_manifest_as_updated(manifest_entry)
manifest_id = manifest_entry.id
return manifest_id
class Orchestrator:
"""
Orchestrator for report processing.
Top level object which is responsible for:
* Maintaining a current list of accounts
* Ensuring that reports are downloaded and processed for all accounts.
"""
def __init__(self, billing_source=None, provider_uuid=None):
"""
Orchestrator for processing.
Args:
billing_source (String): Individual account to retrieve.
"""
self._accounts, self._polling_accounts = self.get_accounts(
billing_source, provider_uuid)
self.worker_cache = WorkerCache()
@staticmethod
def get_accounts(billing_source=None, provider_uuid=None):
"""
Prepare a list of accounts for the orchestrator to get CUR from.
If billing_source is not provided all accounts will be returned, otherwise
only the account for the provided billing_source will be returned.
Still a work in progress, but works for now.
Args:
billing_source (String): Individual account to retrieve.
Returns:
[CostUsageReportAccount] (all), [CostUsageReportAccount] (polling only)
"""
all_accounts = []
polling_accounts = []
try:
all_accounts = AccountsAccessor().get_accounts(provider_uuid)
except AccountsAccessorError as error:
LOG.error("Unable to get accounts. Error: %s", str(error))
if billing_source:
for account in all_accounts:
if billing_source == account.get("billing_source"):
all_accounts = [account]
for account in all_accounts:
if AccountsAccessor().is_polling_account(account):
polling_accounts.append(account)
return all_accounts, polling_accounts
@staticmethod
def get_reports(provider_uuid):
"""
Get months for provider to process.
Args:
(String) provider uuid to determine if initial setup is complete.
Returns:
(List) List of datetime objects.
"""
with ProviderDBAccessor(
provider_uuid=provider_uuid) as provider_accessor:
reports_processed = provider_accessor.get_setup_complete()
if Config.INGEST_OVERRIDE or not reports_processed:
number_of_months = Config.INITIAL_INGEST_NUM_MONTHS
else:
number_of_months = 2
return DateAccessor().get_billing_months(number_of_months)
def start_manifest_processing(self, customer_name, credentials,
data_source, provider_type, schema_name,
provider_uuid, report_month):
"""
Start processing an account's manifest for the specified report_month.
Args:
(String) customer_name - customer name
(String) credentials - credentials object
(String) data_source - report storage location
(String) schema_name - db tenant
(String) provider_uuid - provider unique identifier
(Date) report_month - month to get latest manifest
Returns:
({}) Dictionary containing the following keys:
manifest_id - (String): Manifest ID for ReportManifestDBAccessor
assembly_id - (String): UUID identifying report file
compression - (String): Report compression format
files - ([{"key": full_file_path "local_file": "local file name"}]): List of report files.
"""
downloader = ReportDownloader(
customer_name=customer_name,
credentials=credentials,
data_source=data_source,
provider_type=provider_type,
provider_uuid=provider_uuid,
report_name=None,
)
manifest = downloader.download_manifest(report_month)
if manifest:
LOG.info("Saving all manifest file names.")
record_all_manifest_files(manifest["manifest_id"], [
report.get("local_file")
for report in manifest.get("files", [])
])
LOG.info(f"Found Manifests: {str(manifest)}")
report_files = manifest.get("files", [])
report_tasks = []
for report_file_dict in report_files:
local_file = report_file_dict.get("local_file")
report_file = report_file_dict.get("key")
# Check if report file is complete or in progress.
if record_report_status(manifest["manifest_id"], local_file,
"no_request"):
LOG.info(f"{local_file} was already processed")
continue
cache_key = f"{provider_uuid}:{report_file}"
if self.worker_cache.task_is_running(cache_key):
LOG.info(f"{local_file} process is in progress")
continue
report_context = manifest.copy()
report_context["current_file"] = report_file
report_context["local_file"] = local_file
report_context["key"] = report_file
report_tasks.append(
get_report_files.s(
customer_name,
credentials,
data_source,
provider_type,
schema_name,
provider_uuid,
report_month,
report_context,
))
LOG.info("Download queued - schema_name: %s.", schema_name)
if report_tasks:
async_id = chord(report_tasks, summarize_reports.s())()
LOG.info(f"Manifest Processing Async ID: {async_id}")
return manifest
def prepare(self):
"""
Prepare a processing request for each account.
Scans the database for providers that have reports that need to be processed.
Any report it finds is queued to the appropriate celery task to download
and process those reports.
Args:
None
Returns:
(celery.result.AsyncResult) Async result for download request.
"""
async_result = None
for account in self._polling_accounts:
provider_uuid = account.get("provider_uuid")
report_months = self.get_reports(provider_uuid)
for month in report_months:
LOG.info(
"Getting %s report files for account (provider uuid): %s",
month.strftime("%B %Y"), provider_uuid)
account["report_month"] = month
try:
self.start_manifest_processing(**account)
except ReportDownloaderError as err:
LOG.warning(
f"Unable to download manifest for provider: {provider_uuid}. Error: {str(err)}."
)
continue
except Exception as err:
# Broad exception catching is important here because any errors thrown can
# block all subsequent account processing.
LOG.error(
f"Unexpected manifest processing error for provider: {provider_uuid}. Error: {str(err)}."
)
continue
# update labels
labeler = AccountLabel(
auth=account.get("credentials"),
schema=account.get("schema_name"),
provider_type=account.get("provider_type"),
)
account_number, label = labeler.get_label_details()
if account_number:
LOG.info("Account: %s Label: %s updated.", account_number,
label)
return async_result
def remove_expired_report_data(self,
simulate=False,
line_items_only=False):
"""
Remove expired report data for each account.
Args:
simulate (Boolean) Simulate report data removal
Returns:
(celery.result.AsyncResult) Async result for deletion request.
"""
async_results = []
for account in self._accounts:
LOG.info("Calling remove_expired_data with account: %s", account)
async_result = remove_expired_data.delay(
schema_name=account.get("schema_name"),
provider=account.get("provider_type"),
simulate=simulate,
line_items_only=line_items_only,
)
LOG.info(
"Expired data removal queued - schema_name: %s, Task ID: %s",
account.get("schema_name"),
str(async_result),
)
async_results.append({
"customer": account.get("customer_name"),
"async_id": str(async_result)
})
return async_results
def update_openshift_on_cloud(
self,
schema_name,
openshift_provider_uuid,
infrastructure_provider_uuid,
infrastructure_provider_type,
start_date,
end_date,
manifest_id=None,
queue_name=None,
synchronous=False,
tracing_id=None,
):
"""Update OpenShift on Cloud for a specific OpenShift and cloud source."""
task_name = "masu.processor.tasks.update_openshift_on_cloud"
cache_args = [schema_name, infrastructure_provider_uuid]
if not synchronous:
worker_cache = WorkerCache()
if worker_cache.single_task_is_running(task_name, cache_args):
msg = f"Task {task_name} already running for {cache_args}. Requeuing."
LOG.info(log_json(tracing_id, msg))
update_openshift_on_cloud.s(
schema_name,
openshift_provider_uuid,
infrastructure_provider_uuid,
infrastructure_provider_type,
start_date,
end_date,
manifest_id=manifest_id,
queue_name=queue_name,
synchronous=synchronous,
tracing_id=tracing_id,
).apply_async(queue=queue_name or UPDATE_SUMMARY_TABLES_QUEUE)
return
worker_cache.lock_single_task(task_name,
cache_args,
timeout=settings.WORKER_CACHE_TIMEOUT)
stmt = (f"update_openshift_on_cloud called with args: "
f" schema_name: {schema_name}, "
f" openshift_provider_uuid: {openshift_provider_uuid}, "
f" infrastructure_provider_uuid: {infrastructure_provider_uuid}, "
f" infrastructure_provider_type: {infrastructure_provider_type}, "
f" start_date: {start_date}, "
f" end_date: {end_date}, "
f" manifest_id: {manifest_id}, "
f" queue_name: {queue_name}, "
f" tracing_id: {tracing_id}")
LOG.info(log_json(tracing_id, stmt))
try:
updater = ReportSummaryUpdater(schema_name,
infrastructure_provider_uuid,
manifest_id, tracing_id)
updater.update_openshift_on_cloud_summary_tables(
start_date,
end_date,
openshift_provider_uuid,
infrastructure_provider_uuid,
infrastructure_provider_type,
tracing_id,
)
except ReportSummaryUpdaterCloudError as ex:
LOG.info(
log_json(
tracing_id,
(
f"update_openshift_on_cloud failed for: {infrastructure_provider_type} ",
f"provider: {infrastructure_provider_uuid}, ",
f"OpenShift provider {openshift_provider_uuid}. \nError: {ex}\n",
f"Retry {self.request.retries} of {settings.MAX_UPDATE_RETRIES}",
),
))
raise ReportSummaryUpdaterCloudError
finally:
if not synchronous:
worker_cache.release_single_task(task_name, cache_args)
def get_report_files(self, customer_name, authentication, billing_source,
provider_type, schema_name, provider_uuid, report_month):
"""
Task to download a Report and process the report.
FIXME: A 2 hour timeout is arbitrarily set for in progress processing requests.
Once we know a realistic processing time for the largest CUR file in production
this value can be adjusted or made configurable.
Args:
customer_name (String): Name of the customer owning the cost usage report.
authentication (String): Credential needed to access cost usage report
in the backend provider.
billing_source (String): Location of the cost usage report in the backend provider.
provider_type (String): Koku defined provider type string. Example: Amazon = 'AWS'
schema_name (String): Name of the DB schema
Returns:
None
"""
worker_stats.GET_REPORT_ATTEMPTS_COUNTER.labels(
provider_type=provider_type).inc()
month = report_month
if isinstance(report_month, str):
month = parser.parse(report_month)
cache_key = f"{provider_uuid}:{month}"
reports = _get_report_files(self, customer_name, authentication,
billing_source, provider_type, provider_uuid,
month, cache_key)
stmt = (f"Reports to be processed:\n"
f" schema_name: {customer_name}\n"
f" provider: {provider_type}\n"
f" provider_uuid: {provider_uuid}\n")
for report in reports:
stmt += " file: " + str(report["file"]) + "\n"
LOG.info(stmt[:-1])
reports_to_summarize = []
for report_dict in reports:
with transaction.atomic():
try:
manifest_id = report_dict.get("manifest_id")
file_name = os.path.basename(report_dict.get("file"))
with ReportStatsDBAccessor(file_name, manifest_id) as stats:
started_date = stats.get_last_started_datetime()
completed_date = stats.get_last_completed_datetime()
# Skip processing if already in progress.
if started_date and not completed_date:
expired_start_date = started_date + datetime.timedelta(
hours=Config.REPORT_PROCESSING_TIMEOUT_HOURS)
if DateAccessor().today_with_timezone(
"UTC") < expired_start_date:
LOG.info(
"Skipping processing task for %s since it was started at: %s.",
file_name,
str(started_date),
)
continue
stmt = (f"Processing starting:\n"
f" schema_name: {customer_name}\n"
f" provider: {provider_type}\n"
f" provider_uuid: {provider_uuid}\n"
f' file: {report_dict.get("file")}')
LOG.info(stmt)
worker_stats.PROCESS_REPORT_ATTEMPTS_COUNTER.labels(
provider_type=provider_type).inc()
_process_report_file(schema_name, provider_type, provider_uuid,
report_dict)
known_manifest_ids = [
report.get("manifest_id")
for report in reports_to_summarize
]
if report_dict.get("manifest_id") not in known_manifest_ids:
report_meta = {
"schema_name": schema_name,
"provider_type": provider_type,
"provider_uuid": provider_uuid,
"manifest_id": report_dict.get("manifest_id"),
}
reports_to_summarize.append(report_meta)
except ReportProcessorError as processing_error:
worker_stats.PROCESS_REPORT_ERROR_COUNTER.labels(
provider_type=provider_type).inc()
LOG.error(str(processing_error))
WorkerCache().remove_task_from_cache(cache_key)
raise processing_error
WorkerCache().remove_task_from_cache(cache_key)
return reports_to_summarize
