def setUp(self): """Establish the database connection.""" self._datetime_format = '%Y-%m-%d %H:%M:%S' self._schema = 'acct10001' self.common_accessor = ReportingCommonDBAccessor() self.column_map = self.common_accessor.column_map self.accessor = ReportDBAccessorBase(self._schema, self.column_map) self.report_schema = self.accessor.report_schema print("Connection is successful!")
def setUpClass(cls): """Setup for the class.""" super().setUpClass() cls.aws_tables = list(AWS_CUR_TABLE_MAP.values()) cls.ocp_tables = list(OCP_REPORT_TABLE_MAP.values()) cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) + \ list(OCP_REPORT_TABLE_MAP.values()) report_common_db = ReportingCommonDBAccessor() cls.column_map = report_common_db.column_map report_common_db.close_session()
def setUpClass(cls): """Setup for the class.""" cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) report_common_db = ReportingCommonDBAccessor() column_map = report_common_db.column_map report_common_db.close_session() cls.accessor = ReportDBAccessor(schema='testcustomer', column_map=column_map) cls.creator = ReportObjectCreator( cls.accessor, column_map, cls.accessor.report_schema.column_types )
class ReportingCommonDBAccessorTest(MasuTestCase): """Test Cases for the ReportingCommonDBAccessor object.""" def setUp(self): """Set up the test class with required objects.""" super().setUp() self.accessor = ReportingCommonDBAccessor() self.report_tables = list(AWS_CUR_TABLE_MAP.values()) def test_initializer(self): """Test initializer.""" report_common_schema = self.accessor.report_common_schema self.assertIsInstance(self.accessor.column_map, dict) self.assertTrue(hasattr(report_common_schema, "reporting_common_reportcolumnmap")) def test_generate_column_map(self): """Assert all tables are in the column map.""" column_map = self.accessor.generate_column_map() keys = column_map.keys() tables = copy.deepcopy(self.report_tables) tables.remove(AWS_CUR_TABLE_MAP["cost_entry"]) tables.remove(AWS_CUR_TABLE_MAP["line_item_daily"]) tables.remove(AWS_CUR_TABLE_MAP["line_item_daily_summary"]) tables.remove(AWS_CUR_TABLE_MAP["tags_summary"]) tables.remove(AWS_CUR_TABLE_MAP["ocp_on_aws_daily_summary"]) tables.remove(AWS_CUR_TABLE_MAP["ocp_on_aws_project_daily_summary"]) for table in tables: self.assertIn(table, keys) def test_add(self): """Test the add() function.""" with ReportingCommonDBAccessor() as accessor: accessor._test = Mock() accessor.add("test", {"foo": "bar"})
def __init__(self, schema_name, report_path, compression, provider_uuid): """Initialize base class.""" super().__init__( schema_name=schema_name, report_path=report_path, compression=compression, provider_uuid=provider_uuid, manifest_id=None, processed_report=ProcessedOCPReport(), ) self._report_name = path.basename(report_path) self._cluster_id = report_path.split("/")[-2] self._datetime_format = Config.OCP_DATETIME_STR_FORMAT self._batch_size = Config.REPORT_PROCESSING_BATCH_SIZE with ReportingCommonDBAccessor() as report_common_db: self.column_map = report_common_db.column_map with OCPReportDBAccessor(self._schema, self.column_map) as report_db: self.existing_report_periods_map = report_db.get_report_periods() self.existing_report_map = report_db.get_reports() self.line_item_columns = None
def setUpClass(cls): """Set up the test class with required objects.""" cls.common_accessor = ReportingCommonDBAccessor() cls.column_map = cls.common_accessor.column_map cls.accessor = AWSReportDBAccessor( schema='acct10001', column_map=cls.column_map ) cls.report_schema = cls.accessor.report_schema cls.creator = ReportObjectCreator( cls.accessor, cls.column_map, cls.report_schema.column_types ) cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) cls.foreign_key_tables = [ AWS_CUR_TABLE_MAP['bill'], AWS_CUR_TABLE_MAP['product'], AWS_CUR_TABLE_MAP['pricing'], AWS_CUR_TABLE_MAP['reservation'] ] billing_start = datetime.datetime.utcnow().replace(day=1) cls.manifest_dict = { 'assembly_id': '1234', 'billing_period_start_datetime': billing_start, 'num_total_files': 2, 'provider_id': 1 } cls.manifest_accessor = ReportManifestDBAccessor()
def setUp(self): """Set up a test with database objects.""" super().setUp() self.provider_uuid = '3c6e687e-1a09-4a05-970c-2ccf44b0952e' self.schema = 'acct10001' self.column_map = ReportingCommonDBAccessor().column_map self.creator = ReportObjectCreator(self.schema, self.column_map) reporting_period = self.creator.create_ocp_report_period(provider_uuid=self.provider_uuid) report = self.creator.create_ocp_report(reporting_period) self.creator.create_ocp_usage_line_item(reporting_period, report) self.rates = [ {'metric': {'name': 'cpu_core_usage_per_hour'}, 'tiered_rates': [{'value': 1.5, 'unit': 'USD'}]}, {'metric': {'name': 'memory_gb_usage_per_hour'}, 'tiered_rates': [{'value': 2.5, 'unit': 'USD'}]}, {'metric': {'name': 'cpu_core_request_per_hour'}, 'tiered_rates': [{'value': 3.5, 'unit': 'USD'}]}, {'metric': {'name': 'memory_gb_request_per_hour'}, 'tiered_rates': [{'value': 4.5, 'unit': 'USD'}]}, {'metric': {'name': 'storage_gb_usage_per_month'}, 'tiered_rates': [{'value': 5.5, 'unit': 'USD'}]}, {'metric': {'name': 'storage_gb_request_per_month'}, 'tiered_rates': [{'value': 6.5, 'unit': 'USD'}]}, {'metric': {'name': 'node_cost_per_month'}, 'tiered_rates': [{'value': 7.5, 'unit': 'USD'}]} ] self.markup = {'value': 10, 'unit': 'percent'} self.cost_model = self.creator.create_cost_model( self.provider_uuid, Provider.PROVIDER_OCP, self.rates, self.markup )
def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() with ReportingCommonDBAccessor() as report_common_db: cls.column_map = report_common_db.column_map cls.accessor = OCPReportDBAccessor('acct10001', cls.column_map) cls.report_schema = cls.accessor.report_schema cls.session = cls.accessor._session cls.all_tables = list(OCP_REPORT_TABLE_MAP.values()) cls.creator = ReportObjectCreator(cls.accessor, cls.column_map, cls.report_schema.column_types) cls.date_accessor = DateAccessor() billing_start = cls.date_accessor.today_with_timezone('UTC').replace( day=1) cls.manifest_dict = { 'assembly_id': '1234', 'billing_period_start_datetime': billing_start, 'num_total_files': 2, 'num_processed_files': 2, 'provider_id': 2 } cls.manifest_accessor = ReportManifestDBAccessor()
def setUp(self): """Setup tests.""" super().setUp() self.column_map = ReportingCommonDBAccessor().column_map self.accessor = AWSReportDBAccessor(schema='acct10001', column_map=self.column_map) self.manifest_accessor = ReportManifestDBAccessor()
def setUp(self): """Set up initial data for tests.""" super(TestUploadUtilsWithData, self).setUp() with ReportingCommonDBAccessor(self.schema) as common_accessor: self.column_map = common_accessor.column_map self.creator = ReportObjectCreator(self.schema, self.column_map) timezone = pytz.timezone('UTC') # Arbitrary date as "today" so we don't drift around with `now`. self.today = datetime(2019, 11, 5, 0, 0, 0, tzinfo=timezone) self.today_date = date(year=self.today.year, month=self.today.month, day=self.today.day) self.create_some_data_for_date(self.today) self.yesterday = self.today - timedelta(days=1) self.yesterday_date = date(year=self.yesterday.year, month=self.yesterday.month, day=self.yesterday.day) self.create_some_data_for_date(self.yesterday) self.future = self.today + timedelta(days=900) self.future_date = date(year=self.future.year, month=self.future.month, day=self.future.day)
def setUpClass(cls): """Set up the test class with required objects.""" cls.test_report = './tests/data/test_cur.csv' cls.test_report_gzip = './tests/data/test_cur.csv.gz' cls.processor = AWSReportProcessor( schema_name='acct10001', report_path=cls.test_report, compression=UNCOMPRESSED, provider_id=1, ) cls.date_accessor = DateAccessor() billing_start = cls.date_accessor.today_with_timezone('UTC').replace( year=2018, month=6, day=1, hour=0, minute=0, second=0) cls.manifest_dict = { 'assembly_id': '1234', 'billing_period_start_datetime': billing_start, 'num_total_files': 2, 'provider_id': 1, } cls.manifest_accessor = ReportManifestDBAccessor() with ReportingCommonDBAccessor() as report_common_db: cls.column_map = report_common_db.column_map _report_tables = copy.deepcopy(AWS_CUR_TABLE_MAP) _report_tables.pop('line_item_daily', None) _report_tables.pop('line_item_daily_summary', None) _report_tables.pop('tags_summary', None) cls.report_tables = list(_report_tables.values()) # Grab a single row of test data to work with with open(cls.test_report, 'r') as f: reader = csv.DictReader(f) cls.row = next(reader)
def __init__(self, schema_name, report_path, compression, provider_uuid, manifest_id=None): """Initialize the report processor. Args: schema_name (str): The name of the customer schema to process into report_path (str): Where the report file lives in the file system compression (CONST): How the report file is compressed. Accepted values: UNCOMPRESSED, GZIP_COMPRESSED """ super().__init__( schema_name=schema_name, report_path=report_path, compression=compression, provider_uuid=provider_uuid, manifest_id=manifest_id, processed_report=ProcessedGCPReport(), ) self.line_item_table = GCPCostEntryLineItemDaily() self.line_item_table_name = self.line_item_table._meta.db_table self._report_name = report_path self._batch_size = Config.REPORT_PROCESSING_BATCH_SIZE self._schema = schema_name # Gather database accessors with ReportingCommonDBAccessor() as report_common_db: self.column_map = report_common_db.column_map LOG.info("Initialized report processor for file: %s and schema: %s", report_path, self._schema) self.line_item_columns = None
def setUpClass(cls): """Set up the test class with required objects.""" # These test reports should be replaced with OCP reports once processor is impelmented. cls.test_report = './tests/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv' cls.storage_report = './tests/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_storage.csv' cls.unknown_report = './tests/data/test_cur.csv' cls.test_report_gzip = './tests/data/test_cur.csv.gz' cls.ocp_processor = OCPReportProcessor(schema_name='acct10001', report_path=cls.test_report, compression=UNCOMPRESSED, provider_id=1) with ReportingCommonDBAccessor() as report_common_db: cls.column_map = report_common_db.column_map cls.accessor = OCPReportDBAccessor('acct10001', cls.column_map) cls.report_schema = cls.accessor.report_schema cls.session = cls.accessor._session _report_tables = copy.deepcopy(OCP_REPORT_TABLE_MAP) cls.report_tables = list(_report_tables.values()) # Grab a single row of test data to work with with open(cls.test_report, 'r') as f: reader = csv.DictReader(f) cls.row = next(reader)
def __init__(self, schema_name, report_path, compression): """Initialize the report processor. Args: schema_name (str): The name of the customer schema to process into report_path (str): Where the report file lives in the file system compression (CONST): How the report file is compressed. Accepted values: UNCOMPRESSED, GZIP_COMPRESSED """ if compression.upper() not in ALLOWED_COMPRESSIONS: err_msg = f'Compression {compression} is not supported.' raise MasuProcessingError(err_msg) self._schema_name = schema_name self._report_path = report_path self._compression = compression.upper() self._report_name = path.basename(report_path) self._datetime_format = Config.AWS_DATETIME_STR_FORMAT self._batch_size = Config.REPORT_PROCESSING_BATCH_SIZE self.processed_report = ProcessedReport() # Gather database accessors self.report_common_db = ReportingCommonDBAccessor() self.column_map = self.report_common_db.column_map self.report_common_db.close_session() self.report_db = ReportDBAccessor(schema=self._schema_name, column_map=self.column_map) self.report_schema = self.report_db.report_schema self.temp_table = self.report_db.create_temp_table( AWS_CUR_TABLE_MAP['line_item']) self.line_item_columns = None self.hasher = Hasher(hash_function='sha256') self.hash_columns = self._get_line_item_hash_columns() self.current_bill = self.report_db.get_current_cost_entry_bill() self.existing_cost_entry_map = self.report_db.get_cost_entries() self.existing_product_map = self.report_db.get_products() self.existing_pricing_map = self.report_db.get_pricing() self.existing_reservation_map = self.report_db.get_reservations() LOG.info('Initialized report processor for file: %s and schema: %s', self._report_name, self._schema_name)
def __init__(self, schema): """Establish the database connection. Args: schema (str): The customer schema to associate with """ self._accessor = ReportDBAccessor(schema, ReportingCommonDBAccessor().column_map)
def setUp(self): """Setup the test.""" super().setUp() self.account_id = fake_aws_account_id() self.arn = fake_arn(account_id=self.account_id, region=REGION, service='iam') with ReportingCommonDBAccessor() as common_accessor: self.column_map = common_accessor.column_map
def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() cls.common_accessor = ReportingCommonDBAccessor() cls.column_map = cls.common_accessor.column_map cls.accessor = OCPReportDBAccessor(schema='acct10001', column_map=cls.column_map) cls.report_schema = cls.accessor.report_schema cls.creator = ReportObjectCreator(cls.schema, cls.column_map) cls.all_tables = list(OCP_REPORT_TABLE_MAP.values())
def setUpClass(cls): """Set up for the class.""" super().setUpClass() cls.aws_tables = list(AWS_CUR_TABLE_MAP.values()) cls.ocp_tables = list(OCP_REPORT_TABLE_MAP.values()) cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) + list(OCP_REPORT_TABLE_MAP.values()) with ReportingCommonDBAccessor() as report_common_db: cls.column_map = report_common_db.column_map cls.creator = ReportObjectCreator(cls.schema, cls.column_map)
def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() cls.common_accessor = ReportingCommonDBAccessor() cls.column_map = cls.common_accessor.column_map cls.ocp_provider_uuid = '3c6e687e-1a09-4a05-970c-2ccf44b0952e' cls.accessor = OCPReportDBAccessor(schema='acct10001', column_map=cls.column_map) cls.report_schema = cls.accessor.report_schema cls.creator = ReportObjectCreator(cls.schema, cls.column_map) cls.all_tables = list(OCP_REPORT_TABLE_MAP.values())
def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() cls.test_report_path = "./koku/masu/test/data/gcp/evidence-2019-06-03.csv" cls.date_accessor = DateAccessor() cls.manifest_accessor = ReportManifestDBAccessor() with ReportingCommonDBAccessor() as report_common_db: cls.column_map = report_common_db.column_map
def setUpClass(cls): """Set up the test class with required objects.""" cls.common_accessor = ReportingCommonDBAccessor() cls.column_map = cls.common_accessor.column_map cls.accessor = ReportDBAccessor(schema='testcustomer', column_map=cls.column_map) cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) cls.foreign_key_tables = [ AWS_CUR_TABLE_MAP['bill'], AWS_CUR_TABLE_MAP['product'], AWS_CUR_TABLE_MAP['pricing'], AWS_CUR_TABLE_MAP['reservation'] ]
def setUp(self): super().setUp() self.common_accessor = ReportingCommonDBAccessor() self.column_map = self.common_accessor.column_map self.accessor = OCPReportDBAccessor(schema=self.test_schema, column_map=self.column_map) self.provider_accessor = ProviderDBAccessor( provider_uuid=self.ocp_test_provider_uuid) self.report_schema = self.accessor.report_schema self.creator = ReportObjectCreator(self.accessor, self.column_map, self.report_schema.column_types) self.all_tables = list(OCP_REPORT_TABLE_MAP.values()) self.provider_id = self.provider_accessor.get_provider().id reporting_period = self.creator.create_ocp_report_period( provider_id=self.provider_id) report = self.creator.create_ocp_report( reporting_period, reporting_period.report_period_start) self.creator.create_ocp_usage_line_item(reporting_period, report) self.creator.create_ocp_storage_line_item(reporting_period, report)
def __init__(self, schema, provider): """Establish the database connection. Args: schema (str): The customer schema to associate with """ self._provider = provider self._schema = schema with ReportingCommonDBAccessor() as reporting_common: self._column_map = reporting_common.column_map
def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() with ReportingCommonDBAccessor() as report_common_db: cls.column_map = report_common_db.column_map cls.accessor = AzureReportDBAccessor("acct10001", cls.column_map) cls.report_schema = cls.accessor.report_schema cls.all_tables = list(AZURE_REPORT_TABLE_MAP.values()) cls.creator = ReportObjectCreator(cls.schema, cls.column_map) cls.date_accessor = DateAccessor() cls.manifest_accessor = ReportManifestDBAccessor()
def __init__(self, schema, provider, manifest): """Establish the database connection. Args: schema (str): The customer schema to associate with """ self._schema_name = schema self._provider = provider self._manifest = manifest with ReportingCommonDBAccessor() as reporting_common: self._column_map = reporting_common.column_map self._date_accessor = DateAccessor()
def setUp(self): """Set up the test class with required objects.""" super().setUp() self.common_accessor = ReportingCommonDBAccessor() self.column_map = self.common_accessor.column_map self.accessor = AWSReportDBAccessor(schema=self.schema, column_map=self.column_map) self.all_tables = list(AWS_CUR_TABLE_MAP.values()) self.foreign_key_tables = [ AWS_CUR_TABLE_MAP["bill"], AWS_CUR_TABLE_MAP["product"], AWS_CUR_TABLE_MAP["pricing"], AWS_CUR_TABLE_MAP["reservation"], ]
def get_bills_from_provider(provider_uuid, schema, start_date=None, end_date=None): """ Return the AWS bill IDs given a provider UUID. Args: provider_uuid (str): Provider UUID. schema (str): Tenant schema start_date (datetime, str): Start date for bill IDs. end_date (datetime, str) End date for bill IDs. Returns: (list): AWS cost entry bill objects. """ if isinstance(start_date, datetime.datetime): start_date = start_date.replace(day=1) start_date = start_date.strftime('%Y-%m-%d') elif isinstance(start_date, str): start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d') start_date = start_date.replace(day=1) start_date = start_date.strftime('%Y-%m-%d') if isinstance(end_date, datetime.datetime): end_date = end_date.strftime('%Y-%m-%d') with ReportingCommonDBAccessor() as reporting_common: column_map = reporting_common.column_map with ProviderDBAccessor(provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() if provider.type not in (AMAZON_WEB_SERVICES, AWS_LOCAL_SERVICE_PROVIDER): err_msg = 'Provider UUID is not an AWS type. It is {}'.\ format(provider.type) LOG.warning(err_msg) return [] with AWSReportDBAccessor(schema, column_map) as report_accessor: bill_table_name = AWS_CUR_TABLE_MAP['bill'] bill_obj = getattr(report_accessor.report_schema, bill_table_name) bills = report_accessor.get_cost_entry_bills_query_by_provider( provider.id) if start_date: bills = bills.filter(bill_obj.billing_period_start >= start_date) if end_date: bills = bills.filter(bill_obj.billing_period_start <= end_date) bills = bills.all() return bills
def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() cls.common_accessor = ReportingCommonDBAccessor() cls.column_map = cls.common_accessor.column_map cls.accessor = AzureReportDBAccessor(schema=cls.schema, column_map=cls.column_map) cls.report_schema = cls.accessor.report_schema cls.creator = ReportObjectCreator(cls.schema, cls.column_map) cls.all_tables = list(AZURE_REPORT_TABLE_MAP.values()) cls.foreign_key_tables = [ AZURE_REPORT_TABLE_MAP['bill'], AZURE_REPORT_TABLE_MAP['product'], AZURE_REPORT_TABLE_MAP['meter'], ]
def setUp(self): """Set up a test with database objects.""" super().setUp() self.provider_uuid = '3c6e687e-1a09-4a05-970c-2ccf44b0952e' self.schema = 'acct10001' self.column_map = ReportingCommonDBAccessor().column_map self.creator = ReportObjectCreator(self.schema, self.column_map) reporting_period = self.creator.create_ocp_report_period() report = self.creator.create_ocp_report(reporting_period) self.creator.create_ocp_usage_line_item(reporting_period, report) self.cost_model = self.creator.create_cost_model( self.provider_uuid, 'OCP')
def setUpClass(cls): """Set up the test class with required objects.""" cls.common_accessor = ReportingCommonDBAccessor() cls.column_map = cls.common_accessor.column_map cls.accessor = AWSReportDBAccessor(schema='acct10001', column_map=cls.column_map) cls.report_schema = cls.accessor.report_schema cls.creator = ReportObjectCreator(cls.accessor, cls.column_map, cls.report_schema.column_types) cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) cls.foreign_key_tables = [ AWS_CUR_TABLE_MAP['bill'], AWS_CUR_TABLE_MAP['product'], AWS_CUR_TABLE_MAP['pricing'], AWS_CUR_TABLE_MAP['reservation'] ]