def _get_normalized_packages():
six = Package(name='six', version='1.2')
pip = Package(name='pip', version='20.1')
foo = Package(**{
'name': 'flask',
'version': '0.12',
'dependencies': [
{
'name': 'six',
'version': '1.2'
},
{
'name': 'six',
'version': '1.2'
},
{
'name': 'flask',
'version': '0.12'
}]
})
bar = Package(**{
'name': 'bar',
'version': '0.12',
'dependencies': [Package(**six.dict()), Package(**pip.dict())]
})
return NormalizedPackages([foo, bar], 'pypi')
def test_normalized_packages_basic_direct():
"""Test NormalizedPackages with basic dependency."""
pkg = Package(name='flask', version='0.12')
foo = Package(**{
'name': pkg.name,
'version': pkg.version
})
assert foo is not None
normalized = NormalizedPackages([foo], 'pypi')
assert normalized is not None
assert normalized.direct_dependencies is not None
assert len(normalized.direct_dependencies) == 1
assert pkg in normalized.direct_dependencies
# transtives must be empty
assert len(normalized.transitive_dependencies) == 0
# all must be 1
assert normalized.all_dependencies is not None
assert len(normalized.all_dependencies) == 1
assert pkg in normalized.all_dependencies
# dependency_graph
assert len(normalized.dependency_graph) == 1
assert pkg in normalized.dependency_graph
assert len(normalized.dependency_graph[foo]) == 0
def test_normalized_packages_basic_transitive():
"""Test NormalizedPackages with transitives dependency."""
flask = Package(name='flask', version='0.12')
six = Package(name='six', version='1.2.3')
foo = Package(**{
'name': flask.name,
'version': flask.version,
'dependencies': [{
'name': six.name,
'version': six.version
}]
})
assert foo is not None
normalized = NormalizedPackages([foo], 'pypi')
assert normalized is not None
assert normalized.direct_dependencies is not None
assert len(normalized.direct_dependencies) == 1
assert flask in normalized.direct_dependencies
# transtive should have an entry
assert len(normalized.transitive_dependencies) == 1
assert six in normalized.transitive_dependencies
# all must be 2
assert len(normalized.all_dependencies) == 2
assert flask in normalized.all_dependencies
assert six in normalized.all_dependencies
# dependency graph
assert len(normalized.dependency_graph) == 1
assert len(normalized.dependency_graph[flask]) == 1
assert flask in normalized.dependency_graph
assert six in normalized.dependency_graph[flask]
assert flask not in normalized.dependency_graph[flask]
def test_normalized_packages_golang():
"""Test Normalised Golang Packages."""
pkg = Package(name='github.com/pingcap/tidb/[email protected]/api',
version='v2.0.0-rc.3')
assert pkg is not None
normalized = NormalizedPackages([pkg], Ecosystem.golang)
assert normalized is not None
assert normalized.direct_dependencies is not None
assert len(normalized.direct_dependencies) == 1
# transitives must be empty
assert len(normalized.transitive_dependencies) == 0
def test_perform_license_analysis(_mock1, _mock2):
"""Test license analysis function."""
with open("tests/v2/data/license_analysis.json", "r") as f:
payload = json.load(f)
request = RecommenderRequest(**payload)
comp_graph = License.perform_license_analysis(
packages=NormalizedPackages(request.packages, 'maven'),
filtered_comp_packages_graph=payload['filtered_comp_packages_graph'],
filtered_companion_packages=payload['filtered_companion_packages'],
external_request_id=payload['external_request_id'])
assert comp_graph is not None
def execute(self,
arguments=None,
persist=True,
check_license=False): # noqa: F841
"""Execute task."""
started_at = datetime.datetime.utcnow().strftime(
"%Y-%m-%dT%H:%M:%S.%f")
request = RecommenderRequest(**arguments)
if request.ecosystem != "golang":
normalized_packages = NormalizedPackages(request.packages,
request.ecosystem)
insights_response = self._get_insights_response(
normalized_packages)
companion = self._get_recommended_package_details(
insights_response[0])
else:
companion = []
logging.warning("Recommendation is not yet implemented for golang")
result = StackRecommendationResult(
**arguments,
companion=companion,
)
recommendation = result.dict()
ended_at = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f")
recommendation["_audit"] = {
"started_at": started_at,
"ended_at": ended_at,
"version": "v2",
}
external_request_id = request.external_request_id
if persist:
persist_data_in_db(
external_request_id=external_request_id,
task_result=recommendation,
worker="recommendation_v2",
started_at=started_at,
ended_at=ended_at,
)
logger.info(
"%s Recommendation process completed, result persisted into RDS.",
external_request_id,
)
return {
"recommendation": "success",
"external_request_id": external_request_id,
"result": recommendation,
}
def process_request(request: Dict) -> Aggregator:
"""Task code."""
request = StackAggregatorRequest(**request)
# Always generate registered user report for the given stack, API server
# shall filter the report fields based on registration status.
# This will avoid analysis of stack upon user registration.
if request.ecosystem == 'golang':
normalized_packages = GoNormalizedPackages(request.packages, request.ecosystem)
aggregator = GoAggregator(request, normalized_packages)
else:
normalized_packages = NormalizedPackages(request.packages, request.ecosystem)
aggregator = Aggregator(request, normalized_packages)
aggregator.fetch_details()
return aggregator
def _get_normalized_packages():
six = Package(name='six', version='1.2')
pip = Package(name='pip', version='20.1')
flask = Package(
**{
'name': 'flask',
'version': '0.12',
'dependencies': [{
'name': 'flask-mock',
'version': '0.0.13'
}]
})
bar = Package(**{
'name': 'bar',
'version': '0.12',
'dependencies': [flask, six, pip]
})
return NormalizedPackages([flask, bar], 'pypi')
def test_gremlin_batch_call(_mock_gremlin):
"""Test post_gremlin call according to batch size."""
# empty
_mock_gremlin.return_value = None
packages = NormalizedPackages([], 'pypi')
result = sa.Aggregator(request=StackAggregatorRequest(registration_status="REGISTERED",
uuid="3fa85f64-5717-4562-b3fc-2c963f66afa6",
external_request_id='test_request_id',
ecosystem='pypi', manifest_file_path='/tmp/bin', packages=[_DJANGO]),
normalized_packages=packages).get_package_details_from_graph()
assert result is not None
assert isinstance(result, dict)
_mock_gremlin.assert_not_called()
assert (1, 0, 5) == _gremlin_batch_test(_mock_gremlin, 100)
assert (5, 1, 1) == _gremlin_batch_test(_mock_gremlin, 1)
assert (3, 2, 1) == _gremlin_batch_test(_mock_gremlin, 2)
assert (2, 3, 2) == _gremlin_batch_test(_mock_gremlin, 3)
assert (2, 4, 1) == _gremlin_batch_test(_mock_gremlin, 4)
assert (1, 0, 5) == _gremlin_batch_test(_mock_gremlin, 5)
def _get_insights_response(self, request: RecommenderRequest):
try:
insights_url = ECOSYSTEM_TO_INSIGHTS_URL[request.ecosystem]
except KeyError as e:
raise InsightsNotSupportedEcosystems from e
if not request.packages:
raise InsightsWithEmptyPackageException(
"Request package list is empty")
normalized_packages = NormalizedPackages(request.packages,
request.ecosystem)
package_list = list(
map(lambda epv: epv.name, normalized_packages.direct_dependencies))
insights_payload = InsightsRequest(
ecosystem=normalized_packages.ecosystem,
transitive_stack=list(
map(lambda epv: epv.name,
normalized_packages.transitive_dependencies)),
package_list=package_list,
)
# Call PGM and get the response
return self._call(insights_url, insights_payload)
def execute(self, arguments=None, persist=True, check_license=False):
"""Execute task."""
started_at = datetime.datetime.utcnow().strftime(
"%Y-%m-%dT%H:%M:%S.%f")
request = RecommenderRequest(**arguments)
external_request_id = request.external_request_id
normalized_packages = NormalizedPackages(request.packages,
request.ecosystem)
recommendation = {
'companion': [],
'usage_outliers': [],
}
package_list = [
epv.name for epv in normalized_packages.direct_dependencies
]
if package_list:
insights_payload = {
'ecosystem':
request.ecosystem,
'transitive_stack': [
epv.name
for epv in normalized_packages.transitive_dependencies
],
'unknown_packages_ratio_threshold':
float(os.environ.get('UNKNOWN_PACKAGES_THRESHOLD', 0.3)),
'package_list':
package_list,
'comp_package_count_threshold':
int(os.environ.get('MAX_COMPANION_PACKAGES', 5))
}
if request.ecosystem in self.kronos_ecosystems:
insights_payload.update({
'outlier_probability_threshold':
float(os.environ.get('OUTLIER_THRESHOLD', 0.6)),
'user_persona':
"1", # TODO - remove janus hardcoded value
})
input_task_for_insights_recommender = [insights_payload]
# Call PGM and get the response
insights_response = []
start = time.time()
if request.ecosystem != 'golang':
# No Companion Rec. for Golang.
insights_response = self.call_insights_recommender(
input_task_for_insights_recommender)
logger.info('%s took %0.2f secs for call_insights_recommender()',
external_request_id,
time.time() - start)
# From PGM response process companion and alternate packages and
# then get Data from Graph
# TODO - implement multiple manifest file support for below loop
if insights_response is None:
return {
'recommendation': 'pgm_error',
'external_request_id': external_request_id,
'message': 'PGM Fetching error'
}
for insights_result in insights_response:
companion_packages = []
ecosystem = insights_result['ecosystem']
# Get usage based outliers
recommendation['usage_outliers'] = \
insights_result.get('outlier_package_list', [])
# Append Topics for User Stack
recommendation['input_stack_topics'] = insights_result.get(
'package_to_topic_dict', {})
# Add missing packages unknown to PGM
recommendation['missing_packages_pgm'] = insights_result.get(
'missing_packages', [])
for pkg in insights_result['companion_packages']:
companion_packages.append(pkg['package_name'])
# Get Companion Packages from Graph
graph_request_started_at = time.time()
comp_packages_graph = GraphDB().get_version_information(
companion_packages, ecosystem)
logger.info(
'%s took %0.2f secs for GraphDB().get_version_information()',
external_request_id,
time.time() - graph_request_started_at)
# Apply Version Filters
input_stack = {
epv.name: epv.version
for epv in normalized_packages.direct_dependencies
}
filtered_comp_packages_graph, filtered_list = GraphDB(
).filter_versions(comp_packages_graph,
input_stack,
external_request_id,
rec_type="COMPANION")
filtered_companion_packages = \
set(companion_packages).difference(set(filtered_list))
logger.info('%s Fitered companion packages %s',
external_request_id, filtered_companion_packages)
if check_license:
# Apply License Filters
license_request_started_at = time.time()
lic_filtered_comp_graph = \
License.perform_license_analysis(
packages=normalized_packages,
filtered_comp_packages_graph=filtered_comp_packages_graph,
filtered_companion_packages=filtered_companion_packages,
external_request_id=external_request_id
)
logger.info(
'%s took %0.2f secs for License.perform_license_analysis()',
external_request_id,
time.time() - license_request_started_at)
else:
lic_filtered_comp_graph = filtered_comp_packages_graph
# Get Topics Added to Filtered Packages
topics_comp_packages_graph = GraphDB(). \
get_topics_for_comp(lic_filtered_comp_graph,
insights_result.get('companion_packages', []))
# Create Companion Block
comp_packages = create_package_dict(topics_comp_packages_graph)
final_comp_packages = \
set_valid_cooccurrence_probability(comp_packages)
recommendation['companion'] = final_comp_packages
ended_at = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f")
audit = {
'started_at': started_at,
'ended_at': ended_at,
'version': 'v2'
}
recommendation = StackRecommendationResult(**recommendation,
**request.dict()).dict()
recommendation['_audit'] = audit
if persist:
persist_data_in_db(external_request_id=external_request_id,
task_result=recommendation,
worker='recommendation_v2',
started_at=started_at,
ended_at=ended_at)
logger.info(
'%s Recommendation process completed, result persisted into RDS.',
external_request_id)
return {
'recommendation': 'success',
'external_request_id': external_request_id,
'result': recommendation
}
