def prioritize_dependencies(deps, sdk_type):
"""
Extracts and analyze dependency versions and release dates.
Returns a collection of dependencies which is "high priority" in html format:
1. dependency has major release. e.g org.assertj:assertj-core [2.5.0 -> 3.10.0]
2. dependency is 3 sub-versions behind the newest one. e.g org.tukaani:xz [1.5 -> 1.8]
3. dependency has not been updated for more than 6 months.
Args:
deps: A collection of outdated dependencies.
Return:
high_priority_deps: A collection of dependencies which need to be taken care of before next release.
"""
project_id = ReportGeneratorConfig.GCLOUD_PROJECT_ID
dataset_id = ReportGeneratorConfig.DATASET_ID
table_id = ReportGeneratorConfig.get_bigquery_table_id(sdk_type)
high_priority_deps = []
bigquery_client = BigQueryClientUtils(project_id, dataset_id, table_id)
jira_manager = JiraManager(ReportGeneratorConfig.BEAM_JIRA_HOST,
ReportGeneratorConfig.BEAM_JIRA_BOT_USRENAME,
ReportGeneratorConfig.BEAM_JIRA_BOT_PASSWORD,
ReportGeneratorConfig.get_owners_file(sdk_type))
for dep in deps:
try:
logging.info("\n\nStart processing: " + dep)
dep_name, curr_ver, latest_ver = extract_single_dep(dep)
curr_release_date = None
latest_release_date = None
group_id = None
if sdk_type == 'Java':
# extract the groupid and artifactid
group_id, artifact_id = dep_name.split(":")
dep_details_url = "{0}g:\"{1}\" AND a:\"{2}\"".format(
_MAVEN_CENTRAL_URL, group_id, artifact_id)
curr_release_date = find_release_time_from_maven_central(
group_id, artifact_id, curr_ver)
latest_release_date = find_release_time_from_maven_central(
group_id, artifact_id, latest_ver)
else:
dep_details_url = _PYPI_URL + dep_name
curr_release_date = find_release_time_from_python_compatibility_checking_service(
dep_name, curr_ver)
latest_release_date = find_release_time_from_python_compatibility_checking_service(
dep_name, curr_ver)
if not curr_release_date or not latest_release_date:
curr_release_date, latest_release_date = query_dependency_release_dates_from_bigquery(
bigquery_client, dep_name, curr_ver, latest_ver)
dep_info = """<tr>
<td><a href=\'{0}\'>{1}</a></td>
<td>{2}</td>
<td>{3}</td>
<td>{4}</td>
<td>{5}</td>
</tr>\n""".format(dep_details_url, dep_name, curr_ver, latest_ver,
curr_release_date, latest_release_date)
if compare_dependency_versions(curr_ver, latest_ver):
high_priority_deps.append(dep_info)
jira_manager.run(dep_name,
latest_ver,
sdk_type,
group_id=group_id)
elif compare_dependency_release_dates(curr_release_date,
latest_release_date):
high_priority_deps.append(dep_info)
jira_manager.run(dep_name,
latest_ver,
sdk_type,
group_id=group_id)
except:
traceback.print_exc()
continue
bigquery_client.clean_stale_records_from_table()
return high_priority_deps
def prioritize_dependencies(deps, sdk_type):
"""
Extracts and analyze dependency versions and release dates.
Returns a collection of dependencies which is "high priority" in html format:
1. dependency has major release. e.g org.assertj:assertj-core [2.5.0 -> 3.10.0]
2. dependency is 3 sub-versions behind the newest one. e.g org.tukaani:xz [1.5 -> 1.8]
3. dependency has not been updated for more than 6 months.
Args:
deps: A collection of outdated dependencies.
Return:
high_priority_deps: A collection of dependencies which need to be taken care of before next release.
"""
project_id = ReportGeneratorConfig.GCLOUD_PROJECT_ID
dataset_id = ReportGeneratorConfig.DATASET_ID
table_id = ReportGeneratorConfig.get_bigquery_table_id(sdk_type)
high_priority_deps = []
bigquery_client = BigQueryClientUtils(project_id, dataset_id, table_id)
jira_manager = JiraManager(ReportGeneratorConfig.BEAM_JIRA_HOST,
ReportGeneratorConfig.BEAM_JIRA_BOT_USRENAME,
ReportGeneratorConfig.BEAM_JIRA_BOT_PASSWORD,
ReportGeneratorConfig.get_owners_file(sdk_type))
for dep in deps:
try:
if re.match(r'https?://', dep.lstrip()):
# Gradle-version-plugin's output contains URLs of the libraries
continue
logging.info("\n\nStart processing: " + dep)
dep_name, curr_ver, latest_ver = extract_single_dep(dep)
curr_release_date = None
latest_release_date = None
group_id = None
if sdk_type == 'Java':
# extract the groupid and artifactid
group_id, artifact_id = dep_name.split(":")
dep_details_url = "{0}/{1}/{2}".format(
ReportGeneratorConfig.MAVEN_CENTRAL_URL, group_id,
artifact_id)
curr_release_date = find_release_time_from_maven_central(
group_id, artifact_id, curr_ver)
latest_release_date = find_release_time_from_maven_central(
group_id, artifact_id, latest_ver)
else:
dep_details_url = ReportGeneratorConfig.PYPI_URL + dep_name
curr_release_date = find_release_time_from_python_compatibility_checking_service(
dep_name, curr_ver)
latest_release_date = find_release_time_from_python_compatibility_checking_service(
dep_name, latest_ver)
if not curr_release_date or not latest_release_date:
curr_release_date, latest_release_date = query_dependency_release_dates_from_bigquery(
bigquery_client, dep_name, curr_ver, latest_ver)
dep_info = """<tr>
<td><a href=\'{0}\'>{1}</a></td>
<td>{2}</td>
<td>{3}</td>
<td>{4}</td>
<td>{5}</td>""".format(dep_details_url, dep_name, curr_ver, latest_ver,
curr_release_date, latest_release_date)
if (version_comparer.compare_dependency_versions(
curr_ver, latest_ver) or compare_dependency_release_dates(
curr_release_date, latest_release_date)):
# Create a new issue or update on the existing issue
jira_issue = jira_manager.run(dep_name,
curr_ver,
latest_ver,
sdk_type,
group_id=group_id)
if (jira_issue and jira_issue.fields.status.name
in ['Open', 'Reopened', 'Triage Needed']):
dep_info += "<td><a href=\'{0}\'>{1}</a></td></tr>".format(
ReportGeneratorConfig.BEAM_JIRA_HOST + "browse/" +
jira_issue.key, jira_issue.key)
high_priority_deps.append(dep_info)
except:
traceback.print_exc()
continue
bigquery_client.clean_stale_records_from_table()
return high_priority_deps
def prioritize_dependencies(deps, sdk_type):
"""
Extracts and analyze dependency versions and release dates.
Returns a collection of dependencies which is "high priority" in html format:
1. dependency has major release. e.g org.assertj:assertj-core [2.5.0 -> 3.10.0]
2. dependency is 3 sub-versions behind the newest one. e.g org.tukaani:xz [1.5 -> 1.8]
3. dependency has not been updated for more than 6 months.
Args:
deps: A collection of outdated dependencies.
Return:
high_priority_deps: A collection of dependencies which need to be taken care of before next release.
"""
project_id = ReportGeneratorConfig.GCLOUD_PROJECT_ID
dataset_id = ReportGeneratorConfig.DATASET_ID
table_id = ReportGeneratorConfig.get_bigquery_table_id(sdk_type)
high_priority_deps = []
bigquery_client = BigQueryClientUtils(project_id, dataset_id, table_id)
jira_manager = JiraManager(ReportGeneratorConfig.BEAM_JIRA_HOST,
ReportGeneratorConfig.BEAM_JIRA_BOT_USRENAME,
ReportGeneratorConfig.BEAM_JIRA_BOT_PASSWORD,
ReportGeneratorConfig.get_owners_file(sdk_type))
for dep in deps:
try:
logging.info("\n\nStart processing: " + dep)
dep_name, curr_ver, latest_ver = extract_single_dep(dep)
curr_release_date = None
latest_release_date = None
group_id = None
if sdk_type == 'Java':
# extract the groupid and artifactid
group_id, artifact_id = dep_name.split(":")
dep_details_url = "{0}/{1}/{2}".format(ReportGeneratorConfig.MAVEN_CENTRAL_URL, group_id, artifact_id)
curr_release_date = find_release_time_from_maven_central(group_id, artifact_id, curr_ver)
latest_release_date = find_release_time_from_maven_central(group_id, artifact_id, latest_ver)
else:
dep_details_url = ReportGeneratorConfig.PYPI_URL + dep_name
curr_release_date = find_release_time_from_python_compatibility_checking_service(dep_name, curr_ver)
latest_release_date = find_release_time_from_python_compatibility_checking_service(dep_name, curr_ver)
if not curr_release_date or not latest_release_date:
curr_release_date, latest_release_date = query_dependency_release_dates_from_bigquery(bigquery_client,
dep_name,
curr_ver,
latest_ver)
dep_info = """<tr>
<td><a href=\'{0}\'>{1}</a></td>
<td>{2}</td>
<td>{3}</td>
<td>{4}</td>
<td>{5}</td>""".format(dep_details_url,
dep_name,
curr_ver,
latest_ver,
curr_release_date,
latest_release_date)
if (version_comparer.compare_dependency_versions(curr_ver, latest_ver) or
compare_dependency_release_dates(curr_release_date, latest_release_date)):
# Create a new issue or update on the existing issue
jira_issue = jira_manager.run(dep_name, curr_ver, latest_ver, sdk_type, group_id = group_id)
if (jira_issue.fields.status.name == 'Open' or
jira_issue.fields.status.name == 'Reopened'):
dep_info += "<td><a href=\'{0}\'>{1}</a></td></tr>".format(
ReportGeneratorConfig.BEAM_JIRA_HOST+"browse/"+ jira_issue.key,
jira_issue.key)
high_priority_deps.append(dep_info)
except:
traceback.print_exc()
continue
bigquery_client.clean_stale_records_from_table()
return high_priority_deps
def prioritize_dependencies(deps, sdk_type):
"""
Extracts and analyze dependency versions and release dates.
Returns a collection of dependencies which is "high priority" in html format:
1. dependency has major release. e.g org.assertj:assertj-core [2.5.0 -> 3.10.0]
2. dependency is 3 sub-versions behind the newest one. e.g org.tukaani:xz [1.5 -> 1.8]
3. dependency has not been updated for more than 6 months.
Args:
deps: A collection of outdated dependencies.
Return:
high_priority_deps: A collection of dependencies which need to be taken care of before next release.
"""
project_id = ReportGeneratorConfig.GCLOUD_PROJECT_ID
dataset_id = ReportGeneratorConfig.DATASET_ID
table_id = ReportGeneratorConfig.get_bigquery_table_id(sdk_type)
high_priority_deps = []
bigquery_client = BigQueryClientUtils(project_id, dataset_id, table_id)
jira_manager = JiraManager(ReportGeneratorConfig.BEAM_JIRA_HOST,
ReportGeneratorConfig.BEAM_JIRA_BOT_USRENAME,
ReportGeneratorConfig.BEAM_JIRA_BOT_PASSWORD,
ReportGeneratorConfig.get_owners_file(sdk_type))
for dep in deps:
try:
logging.info("\n\nStart processing: " + dep)
dep_name, curr_ver, latest_ver = extract_single_dep(dep)
curr_release_date, latest_release_date = query_dependency_release_dates(bigquery_client,
dep_name,
curr_ver,
latest_ver)
group_id = None
if sdk_type == 'Java':
# extract the groupid and artifactid
group_id, artifact_id = dep_name.split(":")
dep_details_url = "{0}g:\"{1}\" AND a:\"{2}\"".format(_MAVEN_CENTRAL_URL, group_id, artifact_id)
else:
dep_details_url = _PYPI_URL + dep_name
dep_info = """<tr>
<td><a href=\'{0}\'>{1}</a></td>
<td>{2}</td>
<td>{3}</td>
<td>{4}</td>
<td>{5}</td>
</tr>\n""".format(dep_details_url,
dep_name,
curr_ver,
latest_ver,
curr_release_date,
latest_release_date)
if compare_dependency_versions(curr_ver, latest_ver):
high_priority_deps.append(dep_info)
jira_manager.run(dep_name, latest_ver, sdk_type, group_id = group_id)
elif compare_dependency_release_dates(curr_release_date, latest_release_date):
high_priority_deps.append(dep_info)
jira_manager.run(dep_name, latest_ver, sdk_type, group_id = group_id)
except:
traceback.print_exc()
continue
bigquery_client.clean_stale_records_from_table()
return high_priority_deps