def test_populate_volume_label_summary_table(self):
"""Test that the volume label summary table is populated."""
report_table_name = OCP_REPORT_TABLE_MAP['report']
agg_table_name = OCP_REPORT_TABLE_MAP['volume_label_summary']
report_table = getattr(self.accessor.report_schema, report_table_name)
today = DateAccessor().today_with_timezone('UTC')
last_month = today - relativedelta.relativedelta(months=1)
for start_date in (today, last_month):
period = self.creator.create_ocp_report_period(
self.ocp_provider_uuid, period_date=start_date)
report = self.creator.create_ocp_report(period, start_date)
self.creator.create_ocp_storage_line_item(period, report)
with schema_context(self.schema):
report_entry = report_table.objects.all().aggregate(
Min('interval_start'), Max('interval_start'))
start_date = report_entry['interval_start__min']
end_date = report_entry['interval_start__max']
query = self.accessor._get_db_obj_query(agg_table_name)
with schema_context(self.schema):
initial_count = query.count()
self.accessor.populate_storage_line_item_daily_table(
start_date, end_date, self.cluster_id)
self.accessor.populate_volume_label_summary_table()
self.assertNotEqual(query.count(), initial_count)
with schema_context(self.schema):
tags = query.all()
tag_keys = [tag.key for tag in tags]
with self.accessor._conn.cursor() as cursor:
cursor.execute(
"""SELECT DISTINCT jsonb_object_keys(persistentvolume_labels)
FROM reporting_ocpstoragelineitem_daily""")
expected_tag_keys = cursor.fetchall()
expected_tag_keys = [tag[0] for tag in expected_tag_keys]
self.assertEqual(sorted(tag_keys), sorted(expected_tag_keys))
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.test_report_test_path = "./koku/masu/test/data/test_cur.csv"
cls.test_report_gzip_test_path = "./koku/masu/test/data/test_cur.csv.gz"
cls.date_accessor = DateAccessor()
cls.manifest_accessor = ReportManifestDBAccessor()
_report_tables = copy.deepcopy(AWS_CUR_TABLE_MAP)
_report_tables.pop("line_item_daily", None)
_report_tables.pop("line_item_daily_summary", None)
_report_tables.pop("tags_summary", None)
_report_tables.pop("ocp_on_aws_tags_summary", None)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report_test_path, "r") as f:
reader = csv.DictReader(f)
cls.row = next(reader)
def test_download_file_success_end_date_today(self, mock_bigquery,
mock_makedirs):
"""Assert download_file successful scenario"""
mock_bigquery.client.return_value.query.return_value = ["This", "test"]
end_date = DateAccessor().today().date()
key = f"202011_1234_2020-12-05:{end_date}.csv"
mock_name = "mock-test-customer-end-date"
expected_full_path = f"{DATA_DIR}/{mock_name}/gcp/{key}"
downloader = self.create_gcp_downloader_with_mocked_values(
customer_name=mock_name)
with patch("masu.external.downloader.gcp.gcp_report_downloader.open"):
with patch(
"masu.external.downloader.gcp.gcp_report_downloader.create_daily_archives"
):
full_path, etag, date, _ = downloader.download_file(key)
mock_makedirs.assert_called()
self.assertEqual(etag, self.etag)
self.assertEqual(date, self.today)
self.assertEqual(full_path, expected_full_path)
def setUp(self):
"""Set up each test."""
super().setUp()
billing_start = self.date_accessor.today_with_timezone("UTC").replace(day=1)
self.manifest_dict = {
"assembly_id": "1234",
"billing_period_start_datetime": billing_start,
"num_total_files": 2,
"provider_uuid": self.aws_provider_uuid,
}
self.today = DateAccessor().today_with_timezone("UTC")
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
with ProviderDBAccessor(self.aws_provider_uuid) as provider_accessor:
self.provider = provider_accessor.get_provider()
self.updater = AWSReportSummaryUpdater(self.schema, self.provider, self.manifest)
def test_process_report_inactive_provider(self, mock_get_reports,
mock_summarize):
"""Test processing a report for an inactive provider."""
self.ocp_provider.active = False
self.ocp_provider.save()
mock_download_process_value = [{
'schema_name': self.schema,
'provider_type': OPENSHIFT_CONTAINER_PLATFORM,
'provider_uuid': self.ocp_test_provider_uuid,
'start_date': DateAccessor().today(),
}]
mock_get_reports.return_value = mock_download_process_value
cluster_id = self.ocp_provider_resource_name
sample_report = {'cluster_id': cluster_id}
msg_handler.process_report(sample_report)
mock_summarize.delay.assert_not_called()
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.test_report = './koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv'
cls.date_accessor = DateAccessor()
cls.manifest_accessor = ReportManifestDBAccessor()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
_report_tables = copy.deepcopy(AZURE_REPORT_TABLE_MAP)
_report_tables.pop('line_item_daily_summary', None)
_report_tables.pop('tags_summary', None)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report, 'r', encoding='utf-8-sig') as f:
reader = csv.DictReader(f)
cls.row = next(reader)
def test_manifest_is_ready_is_not_ready(self):
"""Test that False is returned when a manifest is not ready to process."""
billing_start = DateAccessor().today_with_timezone('UTC').replace(day=1)
manifest_dict = {
'assembly_id': '1234',
'billing_period_start_datetime': billing_start,
'num_total_files': 2,
'num_processed_files': 1,
'provider_id': self.ocp_provider.id,
}
with ReportManifestDBAccessor() as accessor:
manifest = accessor.add(**manifest_dict)
manifest_id = manifest.id
updater = ReportSummaryUpdater(
self.schema, self.ocp_test_provider_uuid, manifest_id
)
# manifest_is_ready is now unconditionally returning True, so summary is expected.
self.assertTrue(updater.manifest_is_ready())
def test_populate_volume_label_summary_table(self):
"""Test that the volume label summary table is populated."""
report_table_name = OCP_REPORT_TABLE_MAP['report']
agg_table_name = OCP_REPORT_TABLE_MAP['volume_label_summary']
report_table = getattr(self.accessor.report_schema, report_table_name)
today = DateAccessor().today_with_timezone('UTC')
last_month = today - relativedelta.relativedelta(months=1)
for start_date in (today, last_month):
period = self.creator.create_ocp_report_period(start_date)
report = self.creator.create_ocp_report(period, start_date)
self.creator.create_ocp_storage_line_item(
period,
report
)
start_date, end_date = self.accessor._session.query(
func.min(report_table.interval_start),
func.max(report_table.interval_start)
).first()
query = self.accessor._get_db_obj_query(agg_table_name)
initial_count = query.count()
self.accessor.populate_storage_line_item_daily_table(start_date, end_date, self.cluster_id)
self.accessor.populate_volume_label_summary_table()
self.assertNotEqual(query.count(), initial_count)
tags = query.all()
tag_keys = [tag.key for tag in tags]
self.accessor._cursor.execute(
"""SELECT DISTINCT jsonb_object_keys(persistentvolume_labels)
FROM reporting_ocpstoragelineitem_daily"""
)
expected_tag_keys = self.accessor._cursor.fetchall()
expected_tag_keys = [tag[0] for tag in expected_tag_keys]
self.assertEqual(tag_keys, expected_tag_keys)
def __init__(self, schema, provider, manifest):
"""Initialize the class.
Args:
schema (str) The customer schema to associate with.
provider (Provider db object) Database object for Provider.
manifest (str) The manifest to work with.
Returns:
None
"""
self._schema = schema
self._provider = provider
self._provider_uuid = str(self._provider.uuid)
self._manifest = manifest
with ReportingCommonDBAccessor() as reporting_common:
self._column_map = reporting_common.column_map
self._date_accessor = DateAccessor()
def clean_volume():
"""Clean up the volume in the worker pod."""
LOG.info("Cleaning up the volume at %s " % Config.PVC_DIR)
# get the billing months to use below
months = DateAccessor().get_billing_months(
Config.INITIAL_INGEST_NUM_MONTHS)
db_accessor = ReportManifestDBAccessor()
# this list is initialized with the .gitkeep file so that it does not get deleted
assembly_ids_to_exclude = [".gitkeep"]
# grab the assembly ids to exclude for each month
for month in months:
assembly_ids = db_accessor.get_last_seen_manifest_ids(month)
assembly_ids_to_exclude.extend(assembly_ids)
# now we want to loop through the files and clean up the ones that are not in the exclude list
deleted_files = []
retain_files = []
datehelper = DateHelper()
now = datehelper.now
expiration_date = now - timedelta(seconds=Config.VOLUME_FILE_RETENTION)
for [root, _, filenames] in os.walk(Config.PVC_DIR):
for file in filenames:
match = False
for assembly_id in assembly_ids_to_exclude:
if assembly_id in file:
match = True
# if none of the assembly_ids that we care about were in the filename - we can safely delete it
if not match:
potential_delete = os.path.join(root, file)
if os.path.exists(potential_delete):
file_datetime = datetime.fromtimestamp(
os.path.getmtime(potential_delete))
file_datetime = timezone.make_aware(file_datetime)
if file_datetime < expiration_date:
os.remove(potential_delete)
deleted_files.append(potential_delete)
else:
retain_files.append(potential_delete)
LOG.info("Removing all files older than %s", expiration_date)
LOG.info("The following files were too new to delete: %s", retain_files)
LOG.info("The following files were deleted: %s", deleted_files)
def setUpClass(cls):
"""Set up the test class with required objects."""
cls.test_report = './tests/data/test_cur.csv'
cls.test_report_gzip = './tests/data/test_cur.csv.gz'
cls.provider_id = 1
cls.processor = AWSReportProcessor(
schema_name='acct10001',
report_path=cls.test_report,
compression=UNCOMPRESSED,
provider_id=cls.provider_id
)
cls.date_accessor = DateAccessor()
billing_start = cls.date_accessor.today_with_timezone('UTC').replace(
year=2018,
month=6,
day=1,
hour=0,
minute=0,
second=0
)
cls.assembly_id = '1234'
cls.manifest_dict = {
'assembly_id': cls.assembly_id,
'billing_period_start_datetime': billing_start,
'num_total_files': 2,
'provider_id': cls.provider_id
}
cls.manifest_accessor = ReportManifestDBAccessor()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
_report_tables = copy.deepcopy(AWS_CUR_TABLE_MAP)
_report_tables.pop('line_item_daily', None)
_report_tables.pop('line_item_daily_summary', None)
_report_tables.pop('tags_summary', None)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report, 'r') as f:
reader = csv.DictReader(f)
cls.row = next(reader)
def update_summary_cost_model_costs(self, start_date, end_date):
"""Update the OCP summary table with the charge information.
Args:
start_date (str, Optional) - Start date of range to update derived cost.
end_date (str, Optional) - End date of range to update derived cost.
Returns
None
"""
if isinstance(start_date, str):
start_date = parse(start_date)
if isinstance(end_date, str):
end_date = parse(end_date)
LOG.info(
"Updating cost model costs for \n%s provider: %s (%s). \nCluster ID: %s.",
self._provider.type,
self._provider.name,
self._provider_uuid,
self._cluster_id,
)
self._update_usage_costs(start_date, end_date)
self._update_markup_cost(start_date, end_date)
self._update_monthly_cost(start_date, end_date)
# only update based on tag rates if there are tag rates
# this also lets costs get removed if there is no tiered rate and then add to them if there is a tag_rate
if self._tag_infra_rates != {} or self._tag_supplementary_rates != {}:
self._delete_tag_usage_costs(start_date, end_date, self._provider.uuid)
self._update_tag_usage_costs(start_date, end_date)
self._update_tag_usage_default_costs(start_date, end_date)
self._update_monthly_tag_based_cost(start_date, end_date)
self._update_monthly_tag_based_default_cost(start_date, end_date)
with OCPReportDBAccessor(self._schema) as accessor:
accessor.populate_ui_summary_tables(start_date, end_date, self._provider.uuid)
report_period = accessor.report_periods_for_provider_uuid(self._provider_uuid, start_date)
if report_period:
with schema_context(self._schema):
report_period.derived_cost_datetime = DateAccessor().today_with_timezone("UTC")
report_period.save()
def check_if_manifest_should_be_downloaded(self, assembly_id):
"""Check if we should download this manifest.
We first check if we have a database record of this manifest.
That would indicate that we have already downloaded and at least
begun processing. We then check the last completed time for
a file in this manifest. This second check is to cover the case
when we did not complete processing and need to re-downlaod and
process the manifest.
Returns True if the manifest should be downloaded and processed.
"""
manifest_id = None
num_processed_files = 0
num_total_files = 0
today = DateAccessor().today_with_timezone('UTC')
last_completed_cutoff = today - datetime.timedelta(hours=1)
with ReportManifestDBAccessor() as manifest_accessor:
manifest = manifest_accessor.get_manifest(
assembly_id,
self._provider_id
)
if manifest:
manifest_id = manifest.id
num_processed_files = manifest.num_processed_files
num_total_files = manifest.num_total_files
if num_processed_files < num_total_files:
completed_datetime = manifest_accessor.get_last_report_completed_datetime(
manifest_id
)
if completed_datetime and completed_datetime < last_completed_cutoff:
# It has been more than an hour since we processed a file
# and we didn't finish processing. We should download
# and reprocess.
manifest_accessor.reset_manifest(manifest_id)
return True
# The manifest exists and we have processed all the files.
# We should not redownload.
return False
# The manifest does not exist, this is the first time we are
# downloading and processing it.
return True
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.test_report_path = "./koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv"
cls.test_v2_report_path = "./koku/masu/test/data/azure/azure_version_2.csv"
cls.date_accessor = DateAccessor()
cls.manifest_accessor = ReportManifestDBAccessor()
_report_tables = copy.deepcopy(AZURE_REPORT_TABLE_MAP)
_report_tables.pop("line_item_daily_summary", None)
_report_tables.pop("tags_summary", None)
_report_tables.pop("ocp_on_azure_daily_summary", None)
_report_tables.pop("ocp_on_azure_project_daily_summary", None)
_report_tables.pop("ocp_on_azure_tags_summary", None)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report_path, "r", encoding="utf-8-sig") as f:
reader = csv.DictReader(f)
cls.row = next(reader)
def setUp(self):
""""Set up a test with database objects."""
super().setUp()
if self.accessor._conn.closed:
self.accessor._conn = self.accessor._db.connect()
if self.accessor._pg2_conn.closed:
self.accessor._pg2_conn = self.accessor._get_psycopg2_connection()
if self.accessor._cursor.closed:
self.accessor._cursor = self.accessor._get_psycopg2_cursor()
today = DateAccessor().today_with_timezone('UTC')
bill = self.creator.create_cost_entry_bill(today)
cost_entry = self.creator.create_cost_entry(bill, today)
product = self.creator.create_cost_entry_product()
pricing = self.creator.create_cost_entry_pricing()
reservation = self.creator.create_cost_entry_reservation()
self.creator.create_cost_entry_line_item(bill, cost_entry, product,
pricing, reservation)
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
self.manifest_accessor.commit()
def get_reports(self, number_of_months=2):
"""
Download cost usage reports.
Args:
(Int) Number of monthly reports to download.
Returns:
(List) List of filenames downloaded.
"""
reports = []
try:
current_month = DateAccessor().today().replace(day=1, second=1, microsecond=1)
for month in reversed(range(number_of_months)):
calculated_month = current_month + relativedelta(months=-month)
reports = reports + self.download_report(calculated_month)
except Exception as err:
raise ReportDownloaderError(str(err))
return reports
def test_update_summary_tables_without_manifest(self, mock_daily, mock_sum,
mock_storage_daily,
mock_storage_summary):
"""Test that summary tables are properly run without a manifest."""
# Create an updater that doesn't have a manifest
updater = OCPReportSummaryUpdater(self.schema, self.provider, None)
start_date = DateAccessor().today_with_timezone("UTC")
end_date = start_date + datetime.timedelta(days=1)
bill_date = start_date.replace(day=1).date()
with schema_context(self.schema):
period = self.accessor.get_usage_periods_by_date(bill_date)[0]
period.summary_data_updated_datetime = start_date
period.save()
start_date_str = start_date.strftime("%Y-%m-%d")
end_date_str = end_date.strftime("%Y-%m-%d")
updater.update_daily_tables(start_date_str, end_date_str)
mock_daily.assert_called_with(start_date.date(), end_date.date(),
self.report_period.cluster_id)
mock_storage_daily.assert_called_with(start_date.date(),
end_date.date(),
self.report_period.cluster_id)
mock_sum.assert_not_called()
mock_storage_summary.assert_not_called()
updater.update_summary_tables(start_date_str, end_date_str)
mock_sum.assert_called_with(start_date.date(), end_date.date(),
self.report_period.cluster_id)
mock_storage_summary.assert_called_with(start_date.date(),
end_date.date(),
self.report_period.cluster_id)
with OCPReportDBAccessor(self.schema, self.column_map) as accessor:
period = accessor.get_usage_periods_by_date(bill_date)[0]
self.assertIsNotNone(period.summary_data_creation_datetime)
self.assertGreater(period.summary_data_updated_datetime,
self.today)
def setUp(self):
"""Set up a test with database objects."""
super().setUp()
if self.accessor._cursor.closed:
self.accessor._cursor = self.accessor._get_psycopg2_cursor()
self.cluster_id = 'testcluster'
with schema_context(self.schema):
today = DateAccessor().today_with_timezone('UTC')
bill = self.creator.create_cost_entry_bill(
today, provider_id=self.aws_provider.id)
cost_entry = self.creator.create_cost_entry(bill, today)
product = self.creator.create_cost_entry_product()
pricing = self.creator.create_cost_entry_pricing()
reservation = self.creator.create_cost_entry_reservation()
self.creator.create_cost_entry_line_item(bill, cost_entry, product,
pricing, reservation)
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
# These test reports should be replaced with OCP reports once processor is implemented.
cls.test_report_path = (
"./koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv"
)
cls.storage_report_path = "./koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_storage.csv"
cls.node_report_path = "./koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_node_labels.csv"
cls.namespace_report_path = (
"./koku/masu/test/data/ocp/434eda91-885b-40b2-8733-7a21fad62b56_namespace_labels.csv"
)
cls.unknown_report = "./koku/masu/test/data/test_cur.csv"
cls.test_report_gzip_path = "./koku/masu/test/data/test_cur.csv.gz"
cls.date_accessor = DateAccessor()
cls.billing_start = cls.date_accessor.today_with_timezone(
"UTC").replace(year=2018,
month=6,
day=1,
hour=0,
minute=0,
second=0)
cls.assembly_id = "1234"
cls.manifest_accessor = ReportManifestDBAccessor()
cls.accessor = OCPReportDBAccessor(cls.schema)
cls.report_schema = cls.accessor.report_schema
_report_tables = copy.deepcopy(OCP_REPORT_TABLE_MAP)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report_path) as f:
reader = csv.DictReader(f)
cls.row = next(reader)
with open(cls.storage_report_path) as f:
reader = csv.DictReader(f)
cls.storage_row = next(reader)
def setUp(self):
"""Set up each test."""
super().setUp()
if self.accessor._conn.closed:
self.accessor._conn = self.accessor._db.connect()
if self.accessor._pg2_conn.closed:
self.accessor._pg2_conn = self.accessor._get_psycopg2_connection()
if self.accessor._cursor.closed:
self.accessor._cursor = self.accessor._get_psycopg2_cursor()
self.provider_accessor = ProviderDBAccessor(
provider_uuid=self.aws_test_provider_uuid
)
provider_id = self.provider_accessor.get_provider().id
self.provider_accessor = ProviderDBAccessor(
provider_uuid=self.aws_test_provider_uuid
)
self.updater = AWSReportChargeUpdater(
schema=self.test_schema,
provider_uuid=self.aws_test_provider_uuid,
provider_id=provider_id
)
today = DateAccessor().today_with_timezone('UTC')
bill = self.creator.create_cost_entry_bill(today)
cost_entry = self.creator.create_cost_entry(bill, today)
product = self.creator.create_cost_entry_product()
pricing = self.creator.create_cost_entry_pricing()
reservation = self.creator.create_cost_entry_reservation()
self.creator.create_cost_entry_line_item(
bill,
cost_entry,
product,
pricing,
reservation
)
self.manifest = self.manifest_accessor.add(**self.manifest_dict)
self.manifest_accessor.commit()
with ProviderDBAccessor(self.aws_test_provider_uuid) as provider_accessor:
self.provider = provider_accessor.get_provider()
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.test_report_test_path = './koku/masu/test/data/test_cur.csv'
cls.test_report_gzip_test_path = './koku/masu/test/data/test_cur.csv.gz'
cls.date_accessor = DateAccessor()
cls.manifest_accessor = ReportManifestDBAccessor()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
_report_tables = copy.deepcopy(AWS_CUR_TABLE_MAP)
_report_tables.pop('line_item_daily', None)
_report_tables.pop('line_item_daily_summary', None)
_report_tables.pop('tags_summary', None)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report_test_path, 'r') as f:
reader = csv.DictReader(f)
cls.row = next(reader)
def test_process_file_missing_manifest(self, mock_manifest_accessor, mock_stats_accessor, mock_processor):
"""Test the process_report_file functionality when manifest is missing."""
mock_manifest_accessor.get_manifest_by_id.return_value = None
report_dir = tempfile.mkdtemp()
path = "{}/{}".format(report_dir, "file1.csv")
schema_name = self.schema
provider = Provider.PROVIDER_AWS
provider_uuid = self.aws_provider_uuid
report_dict = {"file": path, "compression": "gzip", "start_date": str(DateAccessor().today())}
mock_proc = mock_processor()
mock_stats_acc = mock_stats_accessor().__enter__()
mock_manifest_acc = mock_manifest_accessor().__enter__()
_process_report_file(schema_name, provider, provider_uuid, report_dict)
mock_proc.process.assert_called()
mock_stats_acc.log_last_started_datetime.assert_called()
mock_stats_acc.log_last_completed_datetime.assert_called()
mock_manifest_acc.mark_manifest_as_updated.assert_not_called()
shutil.rmtree(report_dir)
def test_process_manifest_db_record_race_no_provider(
self, mock_get_manifest):
"""Test that the _process_manifest_db_record returns the correct manifest during a race for initial entry."""
mock_get_manifest.side_effect = [None, None]
side_effect_error = IntegrityError(
"""insert or update on table "reporting_awscostentrybill" violates foreign key constraint "reporting_awscostent_provider_id_a08725b3_fk_api_provi"
DETAIL: Key (provider_id)=(fbe0593a-1b83-4182-b23e-08cd190ed939) is not present in table "api_provider".
"""
# noqa
) # noqa
with patch.object(ReportManifestDBAccessor,
"add",
side_effect=side_effect_error):
downloader = ReportDownloaderBase(
provider_uuid=self.unkown_test_provider_uuid,
cache_key=self.cache_key)
with self.assertRaises(ReportDownloaderError):
downloader._process_manifest_db_record(self.assembly_id,
self.billing_start, 2,
DateAccessor().today())
def test_disk_status_logging_no_dir(self, fake_downloader):
"""Test task for logging when temp directory does not exist."""
logging.disable(logging.NOTSET)
Config.PVC_DIR = '/this/path/does/not/exist'
account = fake_arn(service='iam', generate_account_id=True)
expected = ('INFO:masu.processor._tasks.download:Unable to find' +
f' available disk space. {Config.PVC_DIR} does not exist')
with self.assertLogs('masu.processor._tasks.download',
level='INFO') as logger:
_get_report_files(
Mock(),
customer_name=self.fake.word(),
authentication=account,
provider_type=Provider.PROVIDER_AWS,
report_month=DateAccessor().today(),
provider_uuid=self.aws_provider_uuid,
billing_source=self.fake.word(),
)
self.assertIn(expected, logger.output)
def get_reports(provider_uuid):
"""
Get months for provider to process.
Args:
(String) provider uuid to determine if initial setup is complete.
Returns:
(List) List of datetime objects.
"""
with ProviderDBAccessor(
provider_uuid=provider_uuid) as provider_accessor:
reports_processed = provider_accessor.get_setup_complete()
if Config.INGEST_OVERRIDE or not reports_processed:
number_of_months = Config.INITIAL_INGEST_NUM_MONTHS
else:
number_of_months = 2
return DateAccessor().get_billing_months(number_of_months)
def test_process_file_exception(self, mock_stats_accessor, mock_processor):
"""Test the process_report_file functionality when exception is thrown."""
report_dir = tempfile.mkdtemp()
path = '{}/{}'.format(report_dir, 'file1.csv')
schema_name = self.test_schema
provider = 'AWS'
provider_uuid = self.aws_test_provider_uuid
report_dict = {'file': path,
'compression': 'gzip',
'start_date': str(DateAccessor().today())}
mock_processor.side_effect = ReportProcessorError('mock error')
mock_stats_acc = mock_stats_accessor().__enter__()
with self.assertRaises(ReportProcessorError):
_process_report_file(schema_name, provider, provider_uuid, report_dict)
mock_stats_acc.log_last_started_datetime.assert_called()
mock_stats_acc.log_last_completed_datetime.assert_not_called()
mock_stats_acc.commit.assert_called()
shutil.rmtree(report_dir)
def test_get_manifest_context_for_date(self, mock_manifest, mock_delete):
"""Test that the manifest is read."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
assembly_id = "1234"
compression = "PLAIN"
report_keys = ["file1", "file2"]
mock_manifest.return_value = {
"uuid": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"date": current_month,
"files": report_keys,
}
result = self.ocp_report_downloader.get_manifest_context_for_date(
current_month)
self.assertEqual(result.get("assembly_id"), assembly_id)
self.assertEqual(result.get("compression"), compression)
self.assertIsNotNone(result.get("files"))
def is_backing_off(self):
"""Determine if the provider is waiting to retry."""
def backoff(interval, maximum=64):
"""Exponential back-off."""
return min(maximum,
(2**(interval))) + (random.randint(0, 1000) / 1000.0)
retries = self.get_retries()
timestamp = self.get_timestamp()
backoff_threshold = timestamp + timedelta(
hours=backoff(retries, maximum=24))
LOG.debug(
'[%s] Provider: %s, Retries: %s, Timestamp: %s, Threshold: %s',
self.__class__, self.get_provider_uuid(), retries, timestamp,
backoff_threshold)
if self.get_status() == ProviderStatusCode.WARNING and \
retries <= ProviderStatusAccessor.MAX_RETRIES and \
DateAccessor().today() <= backoff_threshold:
return True
return False
def setUpClass(cls):
"""Set up the test class with required objects."""
# These test reports should be replaced with OCP reports once processor is impelmented.
cls.test_report = './tests/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv'
cls.storage_report = './tests/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_storage.csv'
cls.unknown_report = './tests/data/test_cur.csv'
cls.test_report_gzip = './tests/data/test_cur.csv.gz'
cls.provider_id = 1
cls.ocp_processor = OCPReportProcessor(schema_name='acct10001',
report_path=cls.test_report,
compression=UNCOMPRESSED,
provider_id=cls.provider_id)
cls.date_accessor = DateAccessor()
billing_start = cls.date_accessor.today_with_timezone('UTC').replace(
year=2018, month=6, day=1, hour=0, minute=0, second=0)
cls.assembly_id = '1234'
cls.manifest_dict = {
'assembly_id': cls.assembly_id,
'billing_period_start_datetime': billing_start,
'num_total_files': 2,
'provider_id': 1
}
cls.manifest_accessor = ReportManifestDBAccessor()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
cls.accessor = OCPReportDBAccessor('acct10001', cls.column_map)
cls.report_schema = cls.accessor.report_schema
cls.session = cls.accessor._session
_report_tables = copy.deepcopy(OCP_REPORT_TABLE_MAP)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report, 'r') as f:
reader = csv.DictReader(f)
cls.row = next(reader)
def summarize_reports(reports_to_summarize):
"""
Summarize reports returned from line summary task.
Args:
reports_to_summarize (list) list of reports to process
Returns:
None
"""
for report in reports_to_summarize:
start_date = parser.parse(report.get('start_date'))
start_date = start_date.strftime('%Y-%m-%d')
end_date = DateAccessor().today().strftime('%Y-%m-%d')
LOG.error('report to summarize: %s', str(report))
update_summary_tables.delay(report.get('schema_name'),
report.get('provider_type'),
report.get('provider_uuid'),
start_date,
end_date=end_date,
manifest_id=report.get('manifest_id'))
