def _get_normalized_packages(): six = Package(name='six', version='1.2') pip = Package(name='pip', version='20.1') foo = Package(**{ 'name': 'flask', 'version': '0.12', 'dependencies': [ { 'name': 'six', 'version': '1.2' }, { 'name': 'six', 'version': '1.2' }, { 'name': 'flask', 'version': '0.12' }] }) bar = Package(**{ 'name': 'bar', 'version': '0.12', 'dependencies': [Package(**six.dict()), Package(**pip.dict())] }) return NormalizedPackages([foo, bar], 'pypi')
def test_normalized_packages_basic_direct(): """Test NormalizedPackages with basic dependency.""" pkg = Package(name='flask', version='0.12') foo = Package(**{ 'name': pkg.name, 'version': pkg.version }) assert foo is not None normalized = NormalizedPackages([foo], 'pypi') assert normalized is not None assert normalized.direct_dependencies is not None assert len(normalized.direct_dependencies) == 1 assert pkg in normalized.direct_dependencies # transtives must be empty assert len(normalized.transitive_dependencies) == 0 # all must be 1 assert normalized.all_dependencies is not None assert len(normalized.all_dependencies) == 1 assert pkg in normalized.all_dependencies # dependency_graph assert len(normalized.dependency_graph) == 1 assert pkg in normalized.dependency_graph assert len(normalized.dependency_graph[foo]) == 0
def test_normalized_packages_basic_transitive(): """Test NormalizedPackages with transitives dependency.""" flask = Package(name='flask', version='0.12') six = Package(name='six', version='1.2.3') foo = Package(**{ 'name': flask.name, 'version': flask.version, 'dependencies': [{ 'name': six.name, 'version': six.version }] }) assert foo is not None normalized = NormalizedPackages([foo], 'pypi') assert normalized is not None assert normalized.direct_dependencies is not None assert len(normalized.direct_dependencies) == 1 assert flask in normalized.direct_dependencies # transtive should have an entry assert len(normalized.transitive_dependencies) == 1 assert six in normalized.transitive_dependencies # all must be 2 assert len(normalized.all_dependencies) == 2 assert flask in normalized.all_dependencies assert six in normalized.all_dependencies # dependency graph assert len(normalized.dependency_graph) == 1 assert len(normalized.dependency_graph[flask]) == 1 assert flask in normalized.dependency_graph assert six in normalized.dependency_graph[flask] assert flask not in normalized.dependency_graph[flask]
def test_normalized_packages_golang(): """Test Normalised Golang Packages.""" pkg = Package(name='github.com/pingcap/tidb/[email protected]/api', version='v2.0.0-rc.3') assert pkg is not None normalized = NormalizedPackages([pkg], Ecosystem.golang) assert normalized is not None assert normalized.direct_dependencies is not None assert len(normalized.direct_dependencies) == 1 # transitives must be empty assert len(normalized.transitive_dependencies) == 0
def test_perform_license_analysis(_mock1, _mock2): """Test license analysis function.""" with open("tests/v2/data/license_analysis.json", "r") as f: payload = json.load(f) request = RecommenderRequest(**payload) comp_graph = License.perform_license_analysis( packages=NormalizedPackages(request.packages, 'maven'), filtered_comp_packages_graph=payload['filtered_comp_packages_graph'], filtered_companion_packages=payload['filtered_companion_packages'], external_request_id=payload['external_request_id']) assert comp_graph is not None
def execute(self, arguments=None, persist=True, check_license=False): # noqa: F841 """Execute task.""" started_at = datetime.datetime.utcnow().strftime( "%Y-%m-%dT%H:%M:%S.%f") request = RecommenderRequest(**arguments) if request.ecosystem != "golang": normalized_packages = NormalizedPackages(request.packages, request.ecosystem) insights_response = self._get_insights_response( normalized_packages) companion = self._get_recommended_package_details( insights_response[0]) else: companion = [] logging.warning("Recommendation is not yet implemented for golang") result = StackRecommendationResult( **arguments, companion=companion, ) recommendation = result.dict() ended_at = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f") recommendation["_audit"] = { "started_at": started_at, "ended_at": ended_at, "version": "v2", } external_request_id = request.external_request_id if persist: persist_data_in_db( external_request_id=external_request_id, task_result=recommendation, worker="recommendation_v2", started_at=started_at, ended_at=ended_at, ) logger.info( "%s Recommendation process completed, result persisted into RDS.", external_request_id, ) return { "recommendation": "success", "external_request_id": external_request_id, "result": recommendation, }
def process_request(request: Dict) -> Aggregator: """Task code.""" request = StackAggregatorRequest(**request) # Always generate registered user report for the given stack, API server # shall filter the report fields based on registration status. # This will avoid analysis of stack upon user registration. if request.ecosystem == 'golang': normalized_packages = GoNormalizedPackages(request.packages, request.ecosystem) aggregator = GoAggregator(request, normalized_packages) else: normalized_packages = NormalizedPackages(request.packages, request.ecosystem) aggregator = Aggregator(request, normalized_packages) aggregator.fetch_details() return aggregator
def _get_normalized_packages(): six = Package(name='six', version='1.2') pip = Package(name='pip', version='20.1') flask = Package( **{ 'name': 'flask', 'version': '0.12', 'dependencies': [{ 'name': 'flask-mock', 'version': '0.0.13' }] }) bar = Package(**{ 'name': 'bar', 'version': '0.12', 'dependencies': [flask, six, pip] }) return NormalizedPackages([flask, bar], 'pypi')
def test_gremlin_batch_call(_mock_gremlin): """Test post_gremlin call according to batch size.""" # empty _mock_gremlin.return_value = None packages = NormalizedPackages([], 'pypi') result = sa.Aggregator(request=StackAggregatorRequest(registration_status="REGISTERED", uuid="3fa85f64-5717-4562-b3fc-2c963f66afa6", external_request_id='test_request_id', ecosystem='pypi', manifest_file_path='/tmp/bin', packages=[_DJANGO]), normalized_packages=packages).get_package_details_from_graph() assert result is not None assert isinstance(result, dict) _mock_gremlin.assert_not_called() assert (1, 0, 5) == _gremlin_batch_test(_mock_gremlin, 100) assert (5, 1, 1) == _gremlin_batch_test(_mock_gremlin, 1) assert (3, 2, 1) == _gremlin_batch_test(_mock_gremlin, 2) assert (2, 3, 2) == _gremlin_batch_test(_mock_gremlin, 3) assert (2, 4, 1) == _gremlin_batch_test(_mock_gremlin, 4) assert (1, 0, 5) == _gremlin_batch_test(_mock_gremlin, 5)
def _get_insights_response(self, request: RecommenderRequest): try: insights_url = ECOSYSTEM_TO_INSIGHTS_URL[request.ecosystem] except KeyError as e: raise InsightsNotSupportedEcosystems from e if not request.packages: raise InsightsWithEmptyPackageException( "Request package list is empty") normalized_packages = NormalizedPackages(request.packages, request.ecosystem) package_list = list( map(lambda epv: epv.name, normalized_packages.direct_dependencies)) insights_payload = InsightsRequest( ecosystem=normalized_packages.ecosystem, transitive_stack=list( map(lambda epv: epv.name, normalized_packages.transitive_dependencies)), package_list=package_list, ) # Call PGM and get the response return self._call(insights_url, insights_payload)
def execute(self, arguments=None, persist=True, check_license=False): """Execute task.""" started_at = datetime.datetime.utcnow().strftime( "%Y-%m-%dT%H:%M:%S.%f") request = RecommenderRequest(**arguments) external_request_id = request.external_request_id normalized_packages = NormalizedPackages(request.packages, request.ecosystem) recommendation = { 'companion': [], 'usage_outliers': [], } package_list = [ epv.name for epv in normalized_packages.direct_dependencies ] if package_list: insights_payload = { 'ecosystem': request.ecosystem, 'transitive_stack': [ epv.name for epv in normalized_packages.transitive_dependencies ], 'unknown_packages_ratio_threshold': float(os.environ.get('UNKNOWN_PACKAGES_THRESHOLD', 0.3)), 'package_list': package_list, 'comp_package_count_threshold': int(os.environ.get('MAX_COMPANION_PACKAGES', 5)) } if request.ecosystem in self.kronos_ecosystems: insights_payload.update({ 'outlier_probability_threshold': float(os.environ.get('OUTLIER_THRESHOLD', 0.6)), 'user_persona': "1", # TODO - remove janus hardcoded value }) input_task_for_insights_recommender = [insights_payload] # Call PGM and get the response insights_response = [] start = time.time() if request.ecosystem != 'golang': # No Companion Rec. for Golang. insights_response = self.call_insights_recommender( input_task_for_insights_recommender) logger.info('%s took %0.2f secs for call_insights_recommender()', external_request_id, time.time() - start) # From PGM response process companion and alternate packages and # then get Data from Graph # TODO - implement multiple manifest file support for below loop if insights_response is None: return { 'recommendation': 'pgm_error', 'external_request_id': external_request_id, 'message': 'PGM Fetching error' } for insights_result in insights_response: companion_packages = [] ecosystem = insights_result['ecosystem'] # Get usage based outliers recommendation['usage_outliers'] = \ insights_result.get('outlier_package_list', []) # Append Topics for User Stack recommendation['input_stack_topics'] = insights_result.get( 'package_to_topic_dict', {}) # Add missing packages unknown to PGM recommendation['missing_packages_pgm'] = insights_result.get( 'missing_packages', []) for pkg in insights_result['companion_packages']: companion_packages.append(pkg['package_name']) # Get Companion Packages from Graph graph_request_started_at = time.time() comp_packages_graph = GraphDB().get_version_information( companion_packages, ecosystem) logger.info( '%s took %0.2f secs for GraphDB().get_version_information()', external_request_id, time.time() - graph_request_started_at) # Apply Version Filters input_stack = { epv.name: epv.version for epv in normalized_packages.direct_dependencies } filtered_comp_packages_graph, filtered_list = GraphDB( ).filter_versions(comp_packages_graph, input_stack, external_request_id, rec_type="COMPANION") filtered_companion_packages = \ set(companion_packages).difference(set(filtered_list)) logger.info('%s Fitered companion packages %s', external_request_id, filtered_companion_packages) if check_license: # Apply License Filters license_request_started_at = time.time() lic_filtered_comp_graph = \ License.perform_license_analysis( packages=normalized_packages, filtered_comp_packages_graph=filtered_comp_packages_graph, filtered_companion_packages=filtered_companion_packages, external_request_id=external_request_id ) logger.info( '%s took %0.2f secs for License.perform_license_analysis()', external_request_id, time.time() - license_request_started_at) else: lic_filtered_comp_graph = filtered_comp_packages_graph # Get Topics Added to Filtered Packages topics_comp_packages_graph = GraphDB(). \ get_topics_for_comp(lic_filtered_comp_graph, insights_result.get('companion_packages', [])) # Create Companion Block comp_packages = create_package_dict(topics_comp_packages_graph) final_comp_packages = \ set_valid_cooccurrence_probability(comp_packages) recommendation['companion'] = final_comp_packages ended_at = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f") audit = { 'started_at': started_at, 'ended_at': ended_at, 'version': 'v2' } recommendation = StackRecommendationResult(**recommendation, **request.dict()).dict() recommendation['_audit'] = audit if persist: persist_data_in_db(external_request_id=external_request_id, task_result=recommendation, worker='recommendation_v2', started_at=started_at, ended_at=ended_at) logger.info( '%s Recommendation process completed, result persisted into RDS.', external_request_id) return { 'recommendation': 'success', 'external_request_id': external_request_id, 'result': recommendation }