def setUp(self):
"""Establish the database connection."""
self._datetime_format = '%Y-%m-%d %H:%M:%S'
self._schema = 'acct10001'
self.common_accessor = ReportingCommonDBAccessor()
self.column_map = self.common_accessor.column_map
self.accessor = ReportDBAccessorBase(self._schema, self.column_map)
self.report_schema = self.accessor.report_schema
print("Connection is successful!")
def setUpClass(cls):
"""Setup for the class."""
super().setUpClass()
cls.aws_tables = list(AWS_CUR_TABLE_MAP.values())
cls.ocp_tables = list(OCP_REPORT_TABLE_MAP.values())
cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) + \
list(OCP_REPORT_TABLE_MAP.values())
report_common_db = ReportingCommonDBAccessor()
cls.column_map = report_common_db.column_map
report_common_db.close_session()
def setUpClass(cls):
"""Setup for the class."""
cls.all_tables = list(AWS_CUR_TABLE_MAP.values())
report_common_db = ReportingCommonDBAccessor()
column_map = report_common_db.column_map
report_common_db.close_session()
cls.accessor = ReportDBAccessor(schema='testcustomer',
column_map=column_map)
cls.creator = ReportObjectCreator(
cls.accessor,
column_map,
cls.accessor.report_schema.column_types
)
class ReportingCommonDBAccessorTest(MasuTestCase):
"""Test Cases for the ReportingCommonDBAccessor object."""
def setUp(self):
"""Set up the test class with required objects."""
super().setUp()
self.accessor = ReportingCommonDBAccessor()
self.report_tables = list(AWS_CUR_TABLE_MAP.values())
def test_initializer(self):
"""Test initializer."""
report_common_schema = self.accessor.report_common_schema
self.assertIsInstance(self.accessor.column_map, dict)
self.assertTrue(hasattr(report_common_schema, "reporting_common_reportcolumnmap"))
def test_generate_column_map(self):
"""Assert all tables are in the column map."""
column_map = self.accessor.generate_column_map()
keys = column_map.keys()
tables = copy.deepcopy(self.report_tables)
tables.remove(AWS_CUR_TABLE_MAP["cost_entry"])
tables.remove(AWS_CUR_TABLE_MAP["line_item_daily"])
tables.remove(AWS_CUR_TABLE_MAP["line_item_daily_summary"])
tables.remove(AWS_CUR_TABLE_MAP["tags_summary"])
tables.remove(AWS_CUR_TABLE_MAP["ocp_on_aws_daily_summary"])
tables.remove(AWS_CUR_TABLE_MAP["ocp_on_aws_project_daily_summary"])
for table in tables:
self.assertIn(table, keys)
def test_add(self):
"""Test the add() function."""
with ReportingCommonDBAccessor() as accessor:
accessor._test = Mock()
accessor.add("test", {"foo": "bar"})
def __init__(self, schema_name, report_path, compression, provider_uuid):
"""Initialize base class."""
super().__init__(
schema_name=schema_name,
report_path=report_path,
compression=compression,
provider_uuid=provider_uuid,
manifest_id=None,
processed_report=ProcessedOCPReport(),
)
self._report_name = path.basename(report_path)
self._cluster_id = report_path.split("/")[-2]
self._datetime_format = Config.OCP_DATETIME_STR_FORMAT
self._batch_size = Config.REPORT_PROCESSING_BATCH_SIZE
with ReportingCommonDBAccessor() as report_common_db:
self.column_map = report_common_db.column_map
with OCPReportDBAccessor(self._schema, self.column_map) as report_db:
self.existing_report_periods_map = report_db.get_report_periods()
self.existing_report_map = report_db.get_reports()
self.line_item_columns = None
def setUpClass(cls):
"""Set up the test class with required objects."""
cls.common_accessor = ReportingCommonDBAccessor()
cls.column_map = cls.common_accessor.column_map
cls.accessor = AWSReportDBAccessor(
schema='acct10001',
column_map=cls.column_map
)
cls.report_schema = cls.accessor.report_schema
cls.creator = ReportObjectCreator(
cls.accessor,
cls.column_map,
cls.report_schema.column_types
)
cls.all_tables = list(AWS_CUR_TABLE_MAP.values())
cls.foreign_key_tables = [
AWS_CUR_TABLE_MAP['bill'],
AWS_CUR_TABLE_MAP['product'],
AWS_CUR_TABLE_MAP['pricing'],
AWS_CUR_TABLE_MAP['reservation']
]
billing_start = datetime.datetime.utcnow().replace(day=1)
cls.manifest_dict = {
'assembly_id': '1234',
'billing_period_start_datetime': billing_start,
'num_total_files': 2,
'provider_id': 1
}
cls.manifest_accessor = ReportManifestDBAccessor()
def setUp(self):
"""Set up a test with database objects."""
super().setUp()
self.provider_uuid = '3c6e687e-1a09-4a05-970c-2ccf44b0952e'
self.schema = 'acct10001'
self.column_map = ReportingCommonDBAccessor().column_map
self.creator = ReportObjectCreator(self.schema, self.column_map)
reporting_period = self.creator.create_ocp_report_period(provider_uuid=self.provider_uuid)
report = self.creator.create_ocp_report(reporting_period)
self.creator.create_ocp_usage_line_item(reporting_period, report)
self.rates = [
{'metric': {'name': 'cpu_core_usage_per_hour'},
'tiered_rates': [{'value': 1.5, 'unit': 'USD'}]},
{'metric': {'name': 'memory_gb_usage_per_hour'},
'tiered_rates': [{'value': 2.5, 'unit': 'USD'}]},
{'metric': {'name': 'cpu_core_request_per_hour'},
'tiered_rates': [{'value': 3.5, 'unit': 'USD'}]},
{'metric': {'name': 'memory_gb_request_per_hour'},
'tiered_rates': [{'value': 4.5, 'unit': 'USD'}]},
{'metric': {'name': 'storage_gb_usage_per_month'},
'tiered_rates': [{'value': 5.5, 'unit': 'USD'}]},
{'metric': {'name': 'storage_gb_request_per_month'},
'tiered_rates': [{'value': 6.5, 'unit': 'USD'}]},
{'metric': {'name': 'node_cost_per_month'},
'tiered_rates': [{'value': 7.5, 'unit': 'USD'}]}
]
self.markup = {'value': 10, 'unit': 'percent'}
self.cost_model = self.creator.create_cost_model(
self.provider_uuid, Provider.PROVIDER_OCP, self.rates, self.markup
)
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
cls.accessor = OCPReportDBAccessor('acct10001', cls.column_map)
cls.report_schema = cls.accessor.report_schema
cls.session = cls.accessor._session
cls.all_tables = list(OCP_REPORT_TABLE_MAP.values())
cls.creator = ReportObjectCreator(cls.accessor, cls.column_map,
cls.report_schema.column_types)
cls.date_accessor = DateAccessor()
billing_start = cls.date_accessor.today_with_timezone('UTC').replace(
day=1)
cls.manifest_dict = {
'assembly_id': '1234',
'billing_period_start_datetime': billing_start,
'num_total_files': 2,
'num_processed_files': 2,
'provider_id': 2
}
cls.manifest_accessor = ReportManifestDBAccessor()
def setUp(self):
"""Setup tests."""
super().setUp()
self.column_map = ReportingCommonDBAccessor().column_map
self.accessor = AWSReportDBAccessor(schema='acct10001',
column_map=self.column_map)
self.manifest_accessor = ReportManifestDBAccessor()
def setUp(self):
"""Set up initial data for tests."""
super(TestUploadUtilsWithData, self).setUp()
with ReportingCommonDBAccessor(self.schema) as common_accessor:
self.column_map = common_accessor.column_map
self.creator = ReportObjectCreator(self.schema, self.column_map)
timezone = pytz.timezone('UTC')
# Arbitrary date as "today" so we don't drift around with `now`.
self.today = datetime(2019, 11, 5, 0, 0, 0, tzinfo=timezone)
self.today_date = date(year=self.today.year,
month=self.today.month,
day=self.today.day)
self.create_some_data_for_date(self.today)
self.yesterday = self.today - timedelta(days=1)
self.yesterday_date = date(year=self.yesterday.year,
month=self.yesterday.month,
day=self.yesterday.day)
self.create_some_data_for_date(self.yesterday)
self.future = self.today + timedelta(days=900)
self.future_date = date(year=self.future.year,
month=self.future.month,
day=self.future.day)
def setUpClass(cls):
"""Set up the test class with required objects."""
cls.test_report = './tests/data/test_cur.csv'
cls.test_report_gzip = './tests/data/test_cur.csv.gz'
cls.processor = AWSReportProcessor(
schema_name='acct10001',
report_path=cls.test_report,
compression=UNCOMPRESSED,
provider_id=1,
)
cls.date_accessor = DateAccessor()
billing_start = cls.date_accessor.today_with_timezone('UTC').replace(
year=2018, month=6, day=1, hour=0, minute=0, second=0)
cls.manifest_dict = {
'assembly_id': '1234',
'billing_period_start_datetime': billing_start,
'num_total_files': 2,
'provider_id': 1,
}
cls.manifest_accessor = ReportManifestDBAccessor()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
_report_tables = copy.deepcopy(AWS_CUR_TABLE_MAP)
_report_tables.pop('line_item_daily', None)
_report_tables.pop('line_item_daily_summary', None)
_report_tables.pop('tags_summary', None)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report, 'r') as f:
reader = csv.DictReader(f)
cls.row = next(reader)
def __init__(self, schema_name, report_path, compression, provider_uuid, manifest_id=None):
"""Initialize the report processor.
Args:
schema_name (str): The name of the customer schema to process into
report_path (str): Where the report file lives in the file system
compression (CONST): How the report file is compressed.
Accepted values: UNCOMPRESSED, GZIP_COMPRESSED
"""
super().__init__(
schema_name=schema_name,
report_path=report_path,
compression=compression,
provider_uuid=provider_uuid,
manifest_id=manifest_id,
processed_report=ProcessedGCPReport(),
)
self.line_item_table = GCPCostEntryLineItemDaily()
self.line_item_table_name = self.line_item_table._meta.db_table
self._report_name = report_path
self._batch_size = Config.REPORT_PROCESSING_BATCH_SIZE
self._schema = schema_name
# Gather database accessors
with ReportingCommonDBAccessor() as report_common_db:
self.column_map = report_common_db.column_map
LOG.info("Initialized report processor for file: %s and schema: %s", report_path, self._schema)
self.line_item_columns = None
def setUpClass(cls):
"""Set up the test class with required objects."""
# These test reports should be replaced with OCP reports once processor is impelmented.
cls.test_report = './tests/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv'
cls.storage_report = './tests/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_storage.csv'
cls.unknown_report = './tests/data/test_cur.csv'
cls.test_report_gzip = './tests/data/test_cur.csv.gz'
cls.ocp_processor = OCPReportProcessor(schema_name='acct10001',
report_path=cls.test_report,
compression=UNCOMPRESSED,
provider_id=1)
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
cls.accessor = OCPReportDBAccessor('acct10001', cls.column_map)
cls.report_schema = cls.accessor.report_schema
cls.session = cls.accessor._session
_report_tables = copy.deepcopy(OCP_REPORT_TABLE_MAP)
cls.report_tables = list(_report_tables.values())
# Grab a single row of test data to work with
with open(cls.test_report, 'r') as f:
reader = csv.DictReader(f)
cls.row = next(reader)
def __init__(self, schema_name, report_path, compression):
"""Initialize the report processor.
Args:
schema_name (str): The name of the customer schema to process into
report_path (str): Where the report file lives in the file system
compression (CONST): How the report file is compressed.
Accepted values: UNCOMPRESSED, GZIP_COMPRESSED
"""
if compression.upper() not in ALLOWED_COMPRESSIONS:
err_msg = f'Compression {compression} is not supported.'
raise MasuProcessingError(err_msg)
self._schema_name = schema_name
self._report_path = report_path
self._compression = compression.upper()
self._report_name = path.basename(report_path)
self._datetime_format = Config.AWS_DATETIME_STR_FORMAT
self._batch_size = Config.REPORT_PROCESSING_BATCH_SIZE
self.processed_report = ProcessedReport()
# Gather database accessors
self.report_common_db = ReportingCommonDBAccessor()
self.column_map = self.report_common_db.column_map
self.report_common_db.close_session()
self.report_db = ReportDBAccessor(schema=self._schema_name,
column_map=self.column_map)
self.report_schema = self.report_db.report_schema
self.temp_table = self.report_db.create_temp_table(
AWS_CUR_TABLE_MAP['line_item'])
self.line_item_columns = None
self.hasher = Hasher(hash_function='sha256')
self.hash_columns = self._get_line_item_hash_columns()
self.current_bill = self.report_db.get_current_cost_entry_bill()
self.existing_cost_entry_map = self.report_db.get_cost_entries()
self.existing_product_map = self.report_db.get_products()
self.existing_pricing_map = self.report_db.get_pricing()
self.existing_reservation_map = self.report_db.get_reservations()
LOG.info('Initialized report processor for file: %s and schema: %s',
self._report_name, self._schema_name)
def __init__(self, schema):
"""Establish the database connection.
Args:
schema (str): The customer schema to associate with
"""
self._accessor = ReportDBAccessor(schema,
ReportingCommonDBAccessor().column_map)
def setUp(self):
"""Setup the test."""
super().setUp()
self.account_id = fake_aws_account_id()
self.arn = fake_arn(account_id=self.account_id,
region=REGION,
service='iam')
with ReportingCommonDBAccessor() as common_accessor:
self.column_map = common_accessor.column_map
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.common_accessor = ReportingCommonDBAccessor()
cls.column_map = cls.common_accessor.column_map
cls.accessor = OCPReportDBAccessor(schema='acct10001', column_map=cls.column_map)
cls.report_schema = cls.accessor.report_schema
cls.creator = ReportObjectCreator(cls.schema, cls.column_map)
cls.all_tables = list(OCP_REPORT_TABLE_MAP.values())
def setUpClass(cls):
"""Set up for the class."""
super().setUpClass()
cls.aws_tables = list(AWS_CUR_TABLE_MAP.values())
cls.ocp_tables = list(OCP_REPORT_TABLE_MAP.values())
cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) + list(OCP_REPORT_TABLE_MAP.values())
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
cls.creator = ReportObjectCreator(cls.schema, cls.column_map)
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.common_accessor = ReportingCommonDBAccessor()
cls.column_map = cls.common_accessor.column_map
cls.ocp_provider_uuid = '3c6e687e-1a09-4a05-970c-2ccf44b0952e'
cls.accessor = OCPReportDBAccessor(schema='acct10001', column_map=cls.column_map)
cls.report_schema = cls.accessor.report_schema
cls.creator = ReportObjectCreator(cls.schema, cls.column_map)
cls.all_tables = list(OCP_REPORT_TABLE_MAP.values())
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.test_report_path = "./koku/masu/test/data/gcp/evidence-2019-06-03.csv"
cls.date_accessor = DateAccessor()
cls.manifest_accessor = ReportManifestDBAccessor()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
def setUpClass(cls):
"""Set up the test class with required objects."""
cls.common_accessor = ReportingCommonDBAccessor()
cls.column_map = cls.common_accessor.column_map
cls.accessor = ReportDBAccessor(schema='testcustomer',
column_map=cls.column_map)
cls.all_tables = list(AWS_CUR_TABLE_MAP.values())
cls.foreign_key_tables = [
AWS_CUR_TABLE_MAP['bill'], AWS_CUR_TABLE_MAP['product'],
AWS_CUR_TABLE_MAP['pricing'], AWS_CUR_TABLE_MAP['reservation']
]
def setUp(self):
super().setUp()
self.common_accessor = ReportingCommonDBAccessor()
self.column_map = self.common_accessor.column_map
self.accessor = OCPReportDBAccessor(schema=self.test_schema,
column_map=self.column_map)
self.provider_accessor = ProviderDBAccessor(
provider_uuid=self.ocp_test_provider_uuid)
self.report_schema = self.accessor.report_schema
self.creator = ReportObjectCreator(self.accessor, self.column_map,
self.report_schema.column_types)
self.all_tables = list(OCP_REPORT_TABLE_MAP.values())
self.provider_id = self.provider_accessor.get_provider().id
reporting_period = self.creator.create_ocp_report_period(
provider_id=self.provider_id)
report = self.creator.create_ocp_report(
reporting_period, reporting_period.report_period_start)
self.creator.create_ocp_usage_line_item(reporting_period, report)
self.creator.create_ocp_storage_line_item(reporting_period, report)
def __init__(self, schema, provider):
"""Establish the database connection.
Args:
schema (str): The customer schema to associate with
"""
self._provider = provider
self._schema = schema
with ReportingCommonDBAccessor() as reporting_common:
self._column_map = reporting_common.column_map
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
with ReportingCommonDBAccessor() as report_common_db:
cls.column_map = report_common_db.column_map
cls.accessor = AzureReportDBAccessor("acct10001", cls.column_map)
cls.report_schema = cls.accessor.report_schema
cls.all_tables = list(AZURE_REPORT_TABLE_MAP.values())
cls.creator = ReportObjectCreator(cls.schema, cls.column_map)
cls.date_accessor = DateAccessor()
cls.manifest_accessor = ReportManifestDBAccessor()
def __init__(self, schema, provider, manifest):
"""Establish the database connection.
Args:
schema (str): The customer schema to associate with
"""
self._schema_name = schema
self._provider = provider
self._manifest = manifest
with ReportingCommonDBAccessor() as reporting_common:
self._column_map = reporting_common.column_map
self._date_accessor = DateAccessor()
def setUp(self):
"""Set up the test class with required objects."""
super().setUp()
self.common_accessor = ReportingCommonDBAccessor()
self.column_map = self.common_accessor.column_map
self.accessor = AWSReportDBAccessor(schema=self.schema, column_map=self.column_map)
self.all_tables = list(AWS_CUR_TABLE_MAP.values())
self.foreign_key_tables = [
AWS_CUR_TABLE_MAP["bill"],
AWS_CUR_TABLE_MAP["product"],
AWS_CUR_TABLE_MAP["pricing"],
AWS_CUR_TABLE_MAP["reservation"],
]
def get_bills_from_provider(provider_uuid,
schema,
start_date=None,
end_date=None):
"""
Return the AWS bill IDs given a provider UUID.
Args:
provider_uuid (str): Provider UUID.
schema (str): Tenant schema
start_date (datetime, str): Start date for bill IDs.
end_date (datetime, str) End date for bill IDs.
Returns:
(list): AWS cost entry bill objects.
"""
if isinstance(start_date, datetime.datetime):
start_date = start_date.replace(day=1)
start_date = start_date.strftime('%Y-%m-%d')
elif isinstance(start_date, str):
start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
start_date = start_date.replace(day=1)
start_date = start_date.strftime('%Y-%m-%d')
if isinstance(end_date, datetime.datetime):
end_date = end_date.strftime('%Y-%m-%d')
with ReportingCommonDBAccessor() as reporting_common:
column_map = reporting_common.column_map
with ProviderDBAccessor(provider_uuid) as provider_accessor:
provider = provider_accessor.get_provider()
if provider.type not in (AMAZON_WEB_SERVICES, AWS_LOCAL_SERVICE_PROVIDER):
err_msg = 'Provider UUID is not an AWS type. It is {}'.\
format(provider.type)
LOG.warning(err_msg)
return []
with AWSReportDBAccessor(schema, column_map) as report_accessor:
bill_table_name = AWS_CUR_TABLE_MAP['bill']
bill_obj = getattr(report_accessor.report_schema, bill_table_name)
bills = report_accessor.get_cost_entry_bills_query_by_provider(
provider.id)
if start_date:
bills = bills.filter(bill_obj.billing_period_start >= start_date)
if end_date:
bills = bills.filter(bill_obj.billing_period_start <= end_date)
bills = bills.all()
return bills
def setUpClass(cls):
"""Set up the test class with required objects."""
super().setUpClass()
cls.common_accessor = ReportingCommonDBAccessor()
cls.column_map = cls.common_accessor.column_map
cls.accessor = AzureReportDBAccessor(schema=cls.schema, column_map=cls.column_map)
cls.report_schema = cls.accessor.report_schema
cls.creator = ReportObjectCreator(cls.schema, cls.column_map)
cls.all_tables = list(AZURE_REPORT_TABLE_MAP.values())
cls.foreign_key_tables = [
AZURE_REPORT_TABLE_MAP['bill'],
AZURE_REPORT_TABLE_MAP['product'],
AZURE_REPORT_TABLE_MAP['meter'],
]
def setUp(self):
"""Set up a test with database objects."""
super().setUp()
self.provider_uuid = '3c6e687e-1a09-4a05-970c-2ccf44b0952e'
self.schema = 'acct10001'
self.column_map = ReportingCommonDBAccessor().column_map
self.creator = ReportObjectCreator(self.schema, self.column_map)
reporting_period = self.creator.create_ocp_report_period()
report = self.creator.create_ocp_report(reporting_period)
self.creator.create_ocp_usage_line_item(reporting_period, report)
self.cost_model = self.creator.create_cost_model(
self.provider_uuid, 'OCP')
def setUpClass(cls):
"""Set up the test class with required objects."""
cls.common_accessor = ReportingCommonDBAccessor()
cls.column_map = cls.common_accessor.column_map
cls.accessor = AWSReportDBAccessor(schema='acct10001',
column_map=cls.column_map)
cls.report_schema = cls.accessor.report_schema
cls.creator = ReportObjectCreator(cls.accessor, cls.column_map,
cls.report_schema.column_types)
cls.all_tables = list(AWS_CUR_TABLE_MAP.values())
cls.foreign_key_tables = [
AWS_CUR_TABLE_MAP['bill'], AWS_CUR_TABLE_MAP['product'],
AWS_CUR_TABLE_MAP['pricing'], AWS_CUR_TABLE_MAP['reservation']
]
