def render(self, reports_dict, media_type=None, renderer_context=None): """Render all reports as gzip.""" # pylint: disable=arguments-differ,unused-argument,too-many-locals # pylint: disable=too-many-branches,too-many-statements if not bool(reports_dict): return None files_data = dict() report_id = reports_dict.get('report_id') # Collect Json Data details_json = reports_dict.get('details_json') deployments_json = reports_dict.get('deployments_json') if any(value is None for value in [report_id, details_json, deployments_json]): return None details_name = create_filename('details', 'json', report_id) files_data[details_name] = details_json deployments_name = create_filename('deployments', 'json', report_id) files_data[deployments_name] = deployments_json # Collect CSV Data details_csv = create_details_csv(details_json) deployments_csv = create_deployments_csv(deployments_json) if any(value is None for value in [details_csv, deployments_json]): return None details_csv_name = create_filename('details', 'csv', report_id) files_data[details_csv_name] = details_csv deployments_csv_name = create_filename('deployments', 'csv', report_id) files_data[deployments_csv_name] = deployments_csv tar_buffer = create_tar_buffer(files_data) if tar_buffer is None: logger.error(messages.REPORTS_TAR_ERROR) return None return tar_buffer
def render(self, data, accepted_media_type=None, renderer_context=None): """Render all reports as gzip.""" # pylint: disable=too-many-locals reports_dict = data if not bool(reports_dict): return None report_id = reports_dict.get('report_id') # Collect Json Data details_json = reports_dict.get('details_json') deployments_json = reports_dict.get('deployments_json') if any(value is None for value in [report_id, details_json, deployments_json]): return None # Collect CSV Data details_csv = create_details_csv(details_json) deployments_csv = create_deployments_csv(deployments_json) if any(value is None for value in [details_csv, deployments_csv]): return None # grab hashes details_json_hash = create_hash(details_json, 'json') deployments_json_hash = create_hash(deployments_json, 'json') details_csv_hash = create_hash(details_csv, 'csv') deployments_csv_hash = create_hash(deployments_csv, 'csv') # create the file names details_json_name = create_filename('details', 'json', report_id) deployments_json_name = create_filename( 'deployments', 'json', report_id) details_csv_name = create_filename('details', 'csv', report_id) deployments_csv_name = create_filename('deployments', 'csv', report_id) sha256sum_name = create_filename('SHA256SUM', None, report_id) # map the file names to the file data files_data = { details_json_name: details_json, deployments_json_name: deployments_json, details_csv_name: details_csv, deployments_csv_name: deployments_csv, sha256sum_name: details_json_hash + ' details.json\n' + deployments_json_hash + ' deployments.json\n' + details_csv_hash + ' details.csv\n' + deployments_csv_hash + ' deployments.csv' } tar_buffer = create_tar_buffer(files_data) if tar_buffer is None: logger.error(messages.REPORTS_TAR_ERROR) return None return tar_buffer
def _create_report_slices(report, insights_hosts): """Process facts and convert to fingerprints. :param report: DeploymentReport used to create slices :param insights_hosts: insights host JSON objects :returns: list containing report meta-data and report slices """ # pylint: disable=too-many-locals slice_size_limit = settings.QPC_INSIGHTS_REPORT_SLICE_SIZE number_hosts = len(insights_hosts) if number_hosts % slice_size_limit: number_of_slices = number_hosts // slice_size_limit + 1 hosts_per_slice = number_hosts // number_of_slices + 1 else: number_of_slices = number_hosts // slice_size_limit hosts_per_slice = number_hosts // number_of_slices insights_report_pieces = {} metadata_report_slices = {} source_metadata = { 'report_platform_id': str(report.report_platform_id), 'report_type': 'insights', 'report_version': report.report_version, 'qpc_server_report_id': report.id, 'qpc_server_version': server_version(), 'qpc_server_id': ServerInformation.create_or_retreive_server_id() } metadata = { 'report_id': str(report.report_platform_id), 'host_inventory_api_version': '1.0', 'source': 'qpc', 'source_metadata': source_metadata, 'report_slices': metadata_report_slices } insights_report_pieces[create_filename('metadata', 'json', report.id)] = metadata list_slice_start = 0 list_slice_end = hosts_per_slice for i in range(0, number_hosts, hosts_per_slice): hosts = [] hosts = insights_hosts[list_slice_start:list_slice_end] report_slice_id = str(uuid.uuid4()) report_slice_filename = create_filename(report_slice_id, 'json', report.id) metadata_report_slices[report_slice_id] = {'number_hosts': len(hosts)} report_slice = {'report_slice_id': report_slice_id, 'hosts': hosts} insights_report_pieces[report_slice_filename] = report_slice list_slice_start = list_slice_end if list_slice_end + hosts_per_slice < number_hosts: list_slice_end = hosts_per_slice * (i + 2) else: list_slice_end = number_hosts return Response(insights_report_pieces)
def test_get_insights_report_200_generate_exists(self): """Retrieve insights report.""" url = '/api/v1/reports/1/insights/' self.deployments_report.cached_insights = json.dumps( self.insights_hosts) self.deployments_report.save() with patch('api.insights_report.view.get_object_or_404', return_value=self.deployments_report): response = self.client.get(url) self.assertEqual(response.status_code, 200) response_json = response.json() self.assertIn(create_filename('metadata', 'json', 1), response_json.keys()) for key in response_json: self.assertIn('report_id_1/', key)
def test_get_insights_report_200_generate_exists(self): """Retrieve insights report.""" deployment_report = DeploymentReportFactory( status=DeploymentsReport.STATUS_COMPLETE, ) url = f"/api/v1/reports/{deployment_report.id}/insights/" response = self.client.get(url) self.assertEqual(response.status_code, 200) response_json = response.json() self.assertIn( create_filename("metadata", "json", deployment_report.id), response_json.keys(), ) for key in response_json: self.assertIn(f"report_id_{deployment_report.id}/", key)
def render(self, data, accepted_media_type=None, renderer_context=None): """Render report as json gzip.""" report_dict = data if not bool(report_dict): return None report_id = report_dict.get('report_id') report_type = report_dict.get('report_type') if report_id is None: return None if report_type is None: file_name = '%s.json' % time.strftime('%Y%m%d%H%M%S') else: file_name = create_filename(report_type, 'json', report_id) file_data = {file_name: report_dict} tar_buffer = create_tar_buffer(file_data) return tar_buffer
def test_get_insights_report_200_exists(self): """Retrieve insights report.""" deployment_report = DeploymentReportFactory( number_of_fingerprints=11, status=DeploymentsReport.STATUS_COMPLETE, ) url = f"/api/v1/reports/{deployment_report.id}/insights/" # mock slice size so we can expect 2 slices on this test with override_settings(QPC_INSIGHTS_REPORT_SLICE_SIZE=10): response = self.client.get(url) self.assertEqual(response.status_code, 200, response.json()) response_json = response.json() self.assertIn( create_filename("metadata", "json", deployment_report.id), response_json.keys(), ) report_slices = {} metadata_filename = f"report_id_{deployment_report.id}/metadata.json" for key in response_json: self.assertIn(f"report_id_{deployment_report.id}/", key) if key != metadata_filename: report_slices[key] = response_json[key] # metadata slice number_hosts matches the actual # number of hosts in a slice report_slices_in_metadata = response_json[metadata_filename]["report_slices"] self.assertEqual(len(report_slices_in_metadata), 2) total_returned_hosts_num = 0 for key_1, key_2 in zip(report_slices_in_metadata, report_slices): self.assertEqual(report_slices_in_metadata[key_1]['number_hosts'], len(report_slices[key_2]['hosts'])) # used later to check for the total size total_returned_hosts_num += len(report_slices[key_2]['hosts']) # no hosts lost returned_host_names = { host["bios_uuid"] for slice_key in report_slices # pylint: disable=consider-using-dict-items for host in report_slices[slice_key]["hosts"] } expected_host_names = { host.bios_uuid for host in deployment_report.system_fingerprints.all() } self.assertSetEqual(returned_host_names, expected_host_names) # sum of all hosts in a slice is equal to # the total number of host (before call) self.assertEqual(total_returned_hosts_num, len(expected_host_names))
def render(self, report_dict, media_type=None, renderer_context=None): """Render report as json gzip.""" # pylint: disable=arguments-differ,unused-argument,too-many-locals # pylint: disable=too-many-branches,too-many-statements if not bool(report_dict): return None report_id = report_dict.get('report_id') report_type = report_dict.get('report_type') if report_id is None: return None if report_type is None: file_name = '%s.json' % time.strftime('%Y%m%d%H%M%S') else: file_name = create_filename(report_type, 'json', report_id) file_data = {file_name: report_dict} tar_buffer = create_tar_buffer(file_data) return tar_buffer
def test_get_insights_report_200_exists(self): """Retrieve insights report.""" url = '/api/v1/reports/1/insights/' expected_hosts = [{'unique_id': i} for i in range(10001)] self.deployments_report.cached_insights = json.dumps(expected_hosts) self.deployments_report.save() with patch('api.insights_report.view.get_object_or_404', return_value=self.deployments_report): response = self.client.get(url) self.assertEqual(response.status_code, 200) response_json = response.json() self.assertIn(create_filename('metadata', 'json', 1), response_json.keys()) report_slices = {} for key in response_json: self.assertIn('report_id_1/', key) if key != 'report_id_1/metadata.json': report_slices[key] = response_json[key] # metadata slice number_hosts matches the actual # number of hosts in a slice report_slices_in_metadata = \ response_json['report_id_1/metadata.json']['report_slices'] total_returned_hosts_num = 0 for key_1, key_2 in zip(report_slices_in_metadata, report_slices): self.assertEqual(report_slices_in_metadata[key_1]['number_hosts'], len(report_slices[key_2]['hosts'])) # used later to check for the total size total_returned_hosts_num += len(report_slices[key_2]['hosts']) # no hosts lost returned_host_ids = { host['unique_id'] for slice_key in report_slices for host in report_slices[slice_key]['hosts'] } expected_host_ids = {host['unique_id'] for host in expected_hosts} self.assertSetEqual(returned_host_ids, expected_host_ids) # sum of all hosts in a slice is equal to # the total number of host (before call) self.assertEqual(total_returned_hosts_num, len(expected_hosts))