def update_dependency_dict(dependency_dict, root_pom_id):
root_pom_info = root_pom_id.split(';')
root_pom_group_artifact = root_pom_info[0]
root_pom_group_info = root_pom_group_artifact.split(':')
root_pom_group_id = root_pom_group_info[0]
root_pom_artifact_id = root_pom_group_info[1]
root_pom_version = root_pom_info[1]
root_pom = Pom(root_pom_group_id, root_pom_artifact_id, root_pom_version,
1)
q = queue.Queue()
q.put(root_pom)
visited_pom_dict = {root_pom_group_artifact: root_pom_version}
pom_count = 1
log.info('Added root pom.depth = {}, url = {}.'.format(
root_pom.depth, root_pom.to_url()))
while not q.empty():
pom = q.get()
pom_url = pom.to_url()
log.info('Get dependencies from pom. depth = {}, url = {}.'.format(
pom.depth, pom_url))
try:
tree = elementTree.ElementTree(file=request.urlopen(pom_url))
except HTTPError:
log.warn('Error in open {}'.format(pom_url))
continue
project_element = tree.getroot()
property_dict = {}
parent_element = project_element.find('./maven:parent',
MAVEN_NAME_SPACE)
if parent_element is not None:
# get properties from parent
parent_group_id = parent_element.find(
'./maven:groupId', MAVEN_NAME_SPACE).text.strip(' ${}')
parent_artifact_id = parent_element.find(
'./maven:artifactId', MAVEN_NAME_SPACE).text.strip(' ${}')
parent_version = parent_element.find(
'./maven:version', MAVEN_NAME_SPACE).text.strip(' ${}')
parent_pom = Pom(parent_group_id, parent_artifact_id,
parent_version, pom.depth + 1)
parent_pom_url = parent_pom.to_url()
parent_tree = elementTree.ElementTree(
file=request.urlopen(parent_pom_url))
parent_project_element = parent_tree.getroot()
log.debug(
'Get properties from parent pom. parent_pom_url = {}.'.format(
parent_pom_url))
update_property_dict(parent_project_element, property_dict)
update_property_dict(project_element, property_dict)
# get dependencies
dependency_elements = project_element.findall(
'./maven:dependencyManagement/maven:dependencies/maven:dependency',
MAVEN_NAME_SPACE)
for dependency_element in dependency_elements:
group_id = dependency_element.find(
'./maven:groupId', MAVEN_NAME_SPACE).text.strip(' ${}')
# some group_id contain 'project.groupId', so put project_version first.
if group_id in property_dict:
group_id = property_dict[group_id]
artifact_id = dependency_element.find(
'./maven:artifactId', MAVEN_NAME_SPACE).text.strip(' ')
version = dependency_element.find(
'./maven:version', MAVEN_NAME_SPACE).text.strip(' ${}')
key = group_id + ':' + artifact_id
if version in property_dict:
version = property_dict[version]
if key not in dependency_dict:
dependency_dict[key] = version
log.debug(
'Dependency version added. key = {}, value = {}'.format(
key, version))
elif version != dependency_dict[key]:
log.debug(
'Dependency version skipped. key = {}, version = {}, dependency_dict[key] = {}.'
.format(key, version, dependency_dict[key]))
artifact_type = dependency_element.find('./maven:type',
MAVEN_NAME_SPACE)
artifact_scope = dependency_element.find('./maven:scope',
MAVEN_NAME_SPACE)
if all(
[artifact_type, artifact_scope]) and artifact_type.text.strip(
) == 'pom' and artifact_scope.text.strip() == 'import':
new_pom = Pom(group_id, artifact_id, version, pom.depth + 1)
new_pom_group_artifact = '{}:{}'.format(group_id, artifact_id)
if new_pom_group_artifact not in visited_pom_dict:
q.put(new_pom)
visited_pom_dict[new_pom_group_artifact] = version
pom_count = pom_count + 1
log.debug('Added new pom. depth = {}, url = {}.'.format(
new_pom.depth, new_pom.to_url()))
else:
log.warn(
'Pom exist in visited_pom_dict. depth = {}, url = {}.'.
format(new_pom.depth, new_pom.to_url()))
log.info('Root pom summary. pom_count = {}, root_pom_url = {}'.format(
pom_count, root_pom.to_url()))
return dependency_dict
def update_dependency_dict(dependency_dict, root_pom_id):
root_pom_info = root_pom_id.split(':')
root_pom_group_id = root_pom_info[0]
root_pom_artifact_id = root_pom_info[1]
root_pom_version = get_version_from_external_dependencies(root_pom_id)
root_pom = Pom(root_pom_group_id, root_pom_artifact_id, root_pom_version,
1)
q = queue.Queue()
q.put(root_pom)
pom_count = 1
log.info('Added root pom: {}, depth = {}.'.format(root_pom.to_url(),
root_pom.depth))
while not q.empty():
pom = q.get()
pom_url = pom.to_url()
log.info('Get dependencies from pom: {}, depth = {}.'.format(
pom_url, pom.depth))
tree = elementTree.ElementTree(file=request.urlopen(pom_url))
project_element = tree.getroot()
name_space = {'maven': 'http://maven.apache.org/POM/4.0.0'}
# get properties
properties = project_element.find('maven:properties', name_space)
property_dict = {}
# some property contain 'project.version', so put 'project.version' into property_dict.
version_element = project_element.find('./maven:version', name_space)
if version_element is None:
version_element = project_element.find(
'./maven:parent/maven:version', name_space)
project_version = version_element.text.strip()
property_dict['project.version'] = project_version
# some property contain 'project.groupId', so put 'project.groupId' into property_dict.
group_id_element = project_element.find('./maven:groupId', name_space)
if group_id_element is None:
group_id_element = project_element.find(
'./maven:parent/maven:groupId', name_space)
group_id = group_id_element.text.strip()
property_dict['project.groupId'] = group_id
if properties is not None:
for p in properties:
key = p.tag.split('}', 1)[1]
value = p.text.strip(' ${}')
if value in property_dict:
value = property_dict[value]
property_dict[key] = value
# sometimes project_version contains '${foo}', so update project_version.
if project_version.startswith('${'):
property_dict['project.version'] = property_dict[
project_version.strip(' ${}')]
# get dependencies
dependency_elements = project_element.findall(
'./maven:dependencyManagement/maven:dependencies/maven:dependency',
name_space)
for dependency_element in dependency_elements:
group_id = dependency_element.find('./maven:groupId',
name_space).text.strip(' ${}')
# some group_id contain 'project.groupId', so put project_version first.
if group_id in property_dict:
group_id = property_dict[group_id]
artifact_id = dependency_element.find('./maven:artifactId',
name_space).text.strip(' ')
version = dependency_element.find('./maven:version',
name_space).text.strip(' ${}')
key = group_id + ':' + artifact_id
if version in property_dict:
version = property_dict[version]
if key not in dependency_dict:
dependency_dict[key] = version
log.debug(
'Dependency version added. key = {}, value = {}'.format(
key, version))
elif version != dependency_dict[key]:
log.debug(
'Dependency version skipped. key = {}, version = {}, dependency_dict[key] = {}.'
.format(key, version, dependency_dict[key]))
artifact_type = dependency_element.find('./maven:type', name_space)
if artifact_type is not None and artifact_type.text.strip(
) == 'pom':
new_pom = Pom(group_id, artifact_id, version, pom.depth + 1)
q.put(new_pom)
pom_count = pom_count + 1
log.debug('Added new pom: {}, depth = {}.'.format(
new_pom.to_url(), new_pom.depth))
log.info('Root pom summary: root_pom = {}, pom_count = {}'.format(
root_pom.to_url(), pom_count))
return dependency_dict