def _get_normalized_packages():
    six = Package(name='six', version='1.2')
    pip = Package(name='pip', version='20.1')
    foo = Package(**{
        'name': 'flask',
        'version': '0.12',
        'dependencies': [
            {
                'name': 'six',
                'version': '1.2'
            },
            {
                'name': 'six',
                'version': '1.2'
            },
            {
                'name': 'flask',
                'version': '0.12'
            }]
    })
    bar = Package(**{
        'name': 'bar',
        'version': '0.12',
        'dependencies': [Package(**six.dict()), Package(**pip.dict())]
    })
    return NormalizedPackages([foo, bar], 'pypi')
def test_normalized_packages_basic_direct():
    """Test NormalizedPackages with basic dependency."""
    pkg = Package(name='flask', version='0.12')
    foo = Package(**{
        'name': pkg.name,
        'version': pkg.version
    })
    assert foo is not None
    normalized = NormalizedPackages([foo], 'pypi')
    assert normalized is not None
    assert normalized.direct_dependencies is not None
    assert len(normalized.direct_dependencies) == 1
    assert pkg in normalized.direct_dependencies

    # transtives must be empty
    assert len(normalized.transitive_dependencies) == 0

    # all must be 1
    assert normalized.all_dependencies is not None
    assert len(normalized.all_dependencies) == 1
    assert pkg in normalized.all_dependencies

    # dependency_graph
    assert len(normalized.dependency_graph) == 1
    assert pkg in normalized.dependency_graph
    assert len(normalized.dependency_graph[foo]) == 0
def test_normalized_packages_basic_transitive():
    """Test NormalizedPackages with transitives dependency."""
    flask = Package(name='flask', version='0.12')
    six = Package(name='six', version='1.2.3')
    foo = Package(**{
        'name': flask.name,
        'version': flask.version,
        'dependencies': [{
            'name': six.name,
            'version': six.version
        }]
    })
    assert foo is not None
    normalized = NormalizedPackages([foo], 'pypi')
    assert normalized is not None
    assert normalized.direct_dependencies is not None
    assert len(normalized.direct_dependencies) == 1
    assert flask in normalized.direct_dependencies

    # transtive should have an entry
    assert len(normalized.transitive_dependencies) == 1
    assert six in normalized.transitive_dependencies

    # all must be 2
    assert len(normalized.all_dependencies) == 2
    assert flask in normalized.all_dependencies
    assert six in normalized.all_dependencies

    # dependency graph
    assert len(normalized.dependency_graph) == 1
    assert len(normalized.dependency_graph[flask]) == 1
    assert flask in normalized.dependency_graph
    assert six in normalized.dependency_graph[flask]
    assert flask not in normalized.dependency_graph[flask]
Example #4
0
def test_normalized_packages_golang():
    """Test Normalised Golang Packages."""
    pkg = Package(name='github.com/pingcap/tidb/[email protected]/api',
                  version='v2.0.0-rc.3')
    assert pkg is not None
    normalized = NormalizedPackages([pkg], Ecosystem.golang)
    assert normalized is not None
    assert normalized.direct_dependencies is not None
    assert len(normalized.direct_dependencies) == 1
    # transitives must be empty
    assert len(normalized.transitive_dependencies) == 0
Example #5
0
def test_perform_license_analysis(_mock1, _mock2):
    """Test license analysis function."""
    with open("tests/v2/data/license_analysis.json", "r") as f:
        payload = json.load(f)
    request = RecommenderRequest(**payload)
    comp_graph = License.perform_license_analysis(
        packages=NormalizedPackages(request.packages, 'maven'),
        filtered_comp_packages_graph=payload['filtered_comp_packages_graph'],
        filtered_companion_packages=payload['filtered_companion_packages'],
        external_request_id=payload['external_request_id'])

    assert comp_graph is not None
Example #6
0
    def execute(self,
                arguments=None,
                persist=True,
                check_license=False):  # noqa: F841
        """Execute task."""
        started_at = datetime.datetime.utcnow().strftime(
            "%Y-%m-%dT%H:%M:%S.%f")
        request = RecommenderRequest(**arguments)
        if request.ecosystem != "golang":
            normalized_packages = NormalizedPackages(request.packages,
                                                     request.ecosystem)
            insights_response = self._get_insights_response(
                normalized_packages)
            companion = self._get_recommended_package_details(
                insights_response[0])
        else:
            companion = []
            logging.warning("Recommendation is not yet implemented for golang")

        result = StackRecommendationResult(
            **arguments,
            companion=companion,
        )

        recommendation = result.dict()
        ended_at = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f")
        recommendation["_audit"] = {
            "started_at": started_at,
            "ended_at": ended_at,
            "version": "v2",
        }

        external_request_id = request.external_request_id
        if persist:
            persist_data_in_db(
                external_request_id=external_request_id,
                task_result=recommendation,
                worker="recommendation_v2",
                started_at=started_at,
                ended_at=ended_at,
            )
            logger.info(
                "%s Recommendation process completed, result persisted into RDS.",
                external_request_id,
            )
        return {
            "recommendation": "success",
            "external_request_id": external_request_id,
            "result": recommendation,
        }
    def process_request(request: Dict) -> Aggregator:
        """Task code."""
        request = StackAggregatorRequest(**request)

        # Always generate registered user report for the given stack, API server
        # shall filter the report fields based on registration status.
        # This will avoid analysis of stack upon user registration.
        if request.ecosystem == 'golang':
            normalized_packages = GoNormalizedPackages(request.packages, request.ecosystem)
            aggregator = GoAggregator(request, normalized_packages)
        else:
            normalized_packages = NormalizedPackages(request.packages, request.ecosystem)
            aggregator = Aggregator(request, normalized_packages)
        aggregator.fetch_details()
        return aggregator
def _get_normalized_packages():
    six = Package(name='six', version='1.2')
    pip = Package(name='pip', version='20.1')
    flask = Package(
        **{
            'name': 'flask',
            'version': '0.12',
            'dependencies': [{
                'name': 'flask-mock',
                'version': '0.0.13'
            }]
        })
    bar = Package(**{
        'name': 'bar',
        'version': '0.12',
        'dependencies': [flask, six, pip]
    })
    return NormalizedPackages([flask, bar], 'pypi')
def test_gremlin_batch_call(_mock_gremlin):
    """Test post_gremlin call according to batch size."""
    # empty
    _mock_gremlin.return_value = None
    packages = NormalizedPackages([], 'pypi')
    result = sa.Aggregator(request=StackAggregatorRequest(registration_status="REGISTERED",
                           uuid="3fa85f64-5717-4562-b3fc-2c963f66afa6",
                           external_request_id='test_request_id',
                           ecosystem='pypi', manifest_file_path='/tmp/bin', packages=[_DJANGO]),
                           normalized_packages=packages).get_package_details_from_graph()
    assert result is not None
    assert isinstance(result, dict)
    _mock_gremlin.assert_not_called()
    assert (1, 0, 5) == _gremlin_batch_test(_mock_gremlin, 100)
    assert (5, 1, 1) == _gremlin_batch_test(_mock_gremlin, 1)
    assert (3, 2, 1) == _gremlin_batch_test(_mock_gremlin, 2)
    assert (2, 3, 2) == _gremlin_batch_test(_mock_gremlin, 3)
    assert (2, 4, 1) == _gremlin_batch_test(_mock_gremlin, 4)
    assert (1, 0, 5) == _gremlin_batch_test(_mock_gremlin, 5)
    def _get_insights_response(self, request: RecommenderRequest):
        try:
            insights_url = ECOSYSTEM_TO_INSIGHTS_URL[request.ecosystem]
        except KeyError as e:
            raise InsightsNotSupportedEcosystems from e

        if not request.packages:
            raise InsightsWithEmptyPackageException(
                "Request package list is empty")
        normalized_packages = NormalizedPackages(request.packages,
                                                 request.ecosystem)
        package_list = list(
            map(lambda epv: epv.name, normalized_packages.direct_dependencies))

        insights_payload = InsightsRequest(
            ecosystem=normalized_packages.ecosystem,
            transitive_stack=list(
                map(lambda epv: epv.name,
                    normalized_packages.transitive_dependencies)),
            package_list=package_list,
        )
        # Call PGM and get the response
        return self._call(insights_url, insights_payload)
Example #11
0
    def execute(self, arguments=None, persist=True, check_license=False):
        """Execute task."""
        started_at = datetime.datetime.utcnow().strftime(
            "%Y-%m-%dT%H:%M:%S.%f")
        request = RecommenderRequest(**arguments)
        external_request_id = request.external_request_id

        normalized_packages = NormalizedPackages(request.packages,
                                                 request.ecosystem)

        recommendation = {
            'companion': [],
            'usage_outliers': [],
        }
        package_list = [
            epv.name for epv in normalized_packages.direct_dependencies
        ]
        if package_list:
            insights_payload = {
                'ecosystem':
                request.ecosystem,
                'transitive_stack': [
                    epv.name
                    for epv in normalized_packages.transitive_dependencies
                ],
                'unknown_packages_ratio_threshold':
                float(os.environ.get('UNKNOWN_PACKAGES_THRESHOLD', 0.3)),
                'package_list':
                package_list,
                'comp_package_count_threshold':
                int(os.environ.get('MAX_COMPANION_PACKAGES', 5))
            }
            if request.ecosystem in self.kronos_ecosystems:
                insights_payload.update({
                    'outlier_probability_threshold':
                    float(os.environ.get('OUTLIER_THRESHOLD', 0.6)),
                    'user_persona':
                    "1",  # TODO - remove janus hardcoded value
                })
            input_task_for_insights_recommender = [insights_payload]

            # Call PGM and get the response
            insights_response = []
            start = time.time()
            if request.ecosystem != 'golang':
                # No Companion Rec. for Golang.
                insights_response = self.call_insights_recommender(
                    input_task_for_insights_recommender)

            logger.info('%s took %0.2f secs for call_insights_recommender()',
                        external_request_id,
                        time.time() - start)

            # From PGM response process companion and alternate packages and
            # then get Data from Graph
            # TODO - implement multiple manifest file support for below loop

            if insights_response is None:
                return {
                    'recommendation': 'pgm_error',
                    'external_request_id': external_request_id,
                    'message': 'PGM Fetching error'
                }

            for insights_result in insights_response:
                companion_packages = []
                ecosystem = insights_result['ecosystem']

                # Get usage based outliers
                recommendation['usage_outliers'] = \
                    insights_result.get('outlier_package_list', [])

                # Append Topics for User Stack
                recommendation['input_stack_topics'] = insights_result.get(
                    'package_to_topic_dict', {})
                # Add missing packages unknown to PGM
                recommendation['missing_packages_pgm'] = insights_result.get(
                    'missing_packages', [])
                for pkg in insights_result['companion_packages']:
                    companion_packages.append(pkg['package_name'])

                # Get Companion Packages from Graph
                graph_request_started_at = time.time()
                comp_packages_graph = GraphDB().get_version_information(
                    companion_packages, ecosystem)
                logger.info(
                    '%s took %0.2f secs for GraphDB().get_version_information()',
                    external_request_id,
                    time.time() - graph_request_started_at)

                # Apply Version Filters
                input_stack = {
                    epv.name: epv.version
                    for epv in normalized_packages.direct_dependencies
                }
                filtered_comp_packages_graph, filtered_list = GraphDB(
                ).filter_versions(comp_packages_graph,
                                  input_stack,
                                  external_request_id,
                                  rec_type="COMPANION")

                filtered_companion_packages = \
                    set(companion_packages).difference(set(filtered_list))
                logger.info('%s Fitered companion packages %s',
                            external_request_id, filtered_companion_packages)

                if check_license:
                    # Apply License Filters
                    license_request_started_at = time.time()
                    lic_filtered_comp_graph = \
                        License.perform_license_analysis(
                            packages=normalized_packages,
                            filtered_comp_packages_graph=filtered_comp_packages_graph,
                            filtered_companion_packages=filtered_companion_packages,
                            external_request_id=external_request_id
                        )
                    logger.info(
                        '%s took %0.2f secs for License.perform_license_analysis()',
                        external_request_id,
                        time.time() - license_request_started_at)
                else:
                    lic_filtered_comp_graph = filtered_comp_packages_graph

                # Get Topics Added to Filtered Packages
                topics_comp_packages_graph = GraphDB(). \
                    get_topics_for_comp(lic_filtered_comp_graph,
                                        insights_result.get('companion_packages', []))

                # Create Companion Block
                comp_packages = create_package_dict(topics_comp_packages_graph)
                final_comp_packages = \
                    set_valid_cooccurrence_probability(comp_packages)

                recommendation['companion'] = final_comp_packages

        ended_at = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f")
        audit = {
            'started_at': started_at,
            'ended_at': ended_at,
            'version': 'v2'
        }

        recommendation = StackRecommendationResult(**recommendation,
                                                   **request.dict()).dict()
        recommendation['_audit'] = audit

        if persist:
            persist_data_in_db(external_request_id=external_request_id,
                               task_result=recommendation,
                               worker='recommendation_v2',
                               started_at=started_at,
                               ended_at=ended_at)
            logger.info(
                '%s Recommendation process completed, result persisted into RDS.',
                external_request_id)

        return {
            'recommendation': 'success',
            'external_request_id': external_request_id,
            'result': recommendation
        }