def add_paths_property(
properties: dict, repo_node_id: int, package_name: str, neo4j: Neo4j):
"""Add path names as properties based on search.
Search a git repository and add file names which contain matches to an
:IMPLEMENTED_BY relationship matched agains package_name and repoe_node_id.
:param dict properties:
Mapping of property name to propertie values to be added to
relationship.
:param int repo_node_id:
Identifier for :GitHubRepository node which the :IMPLEMENTED_BY
relationship points to.
:param str package_name:
Package name of :App node.
:param Neo4j neo4j:
Neo4j instance to add nodes to.
"""
parameters = {
'package': package_name,
'repo_id': repo_node_id,
'rel_properties': properties,
}
query = '''
MATCH
(a:App {id: {package}}), (repo:GitHubRepository)
WHERE id(repo) = {repo_id}
MERGE (a)-[r:IMPLEMENTED_BY]->(repo)
ON CREATE SET r = {rel_properties}
ON MATCH SET r += {rel_properties}
'''
neo4j.run(query, **parameters)
def add_branche_nodes(branches: List[dict], repo_node_id: int, neo4j: Neo4j):
"""Create nodes representing GIT branches of a repository.
Creates a node for each branch and links it with the repository identified
by repo_node_id and the commit the branch points to.
:param List[Dict[str, str]] branches:
List of information on branches.
:param int repo_node_id:
ID of node the branches should be linked to.
:param Neo4j neo4j:
Neo4j instance to add nodes to.
"""
for branch in branches:
parameters = {
'commit_hash': branch.get('commit_hash'),
'repo_id': repo_node_id,
'branch_details': {
'name': branch.get('branch_name'),
},
}
neo4j.run(
'''
MATCH (repo:GitHubRepository) WHERE id(repo) = {repo_id}
MERGE (commit:Commit {id: {commit_hash}})
CREATE
(branch:Branch {branch_details})-[:BELONGS_TO]->(repo),
(branch)-[:POINTS_TO]->(commit)
''', **parameters)
def add_tag_nodes(tags: List[dict], repo_node_id: int, neo4j: Neo4j):
"""Create nodes representing GIT tags of a repository.
Creates a node for each tag and links it with the repository identified
by repo_node_id and the commit the tag points to.
:param List[Dict[str, str]] tags:
List of tag data.
:param int repo_node_id:
ID of node the tags should be linked to.
:param Neo4j neo4j:
Neo4j instance to add nodes to.
"""
for tag in tags:
parameters = {
'commit_hash': tag.get('commit_hash'),
'repo_id': repo_node_id,
'tag_details': {
'name': tag.get('tag_name'),
'message': tag.get('tag_message'),
},
}
neo4j.run(
'''
MATCH (repo:GitHubRepository) WHERE id(repo) = {repo_id}
MERGE (commit:Commit {id: {commit_hash}})
CREATE
(tag:Tag {tag_details})-[:BELONGS_TO]->(repo),
(tag)-[:POINTS_TO]->(commit)
''', **parameters)
def add_fork_relationships(neo4j: Neo4j):
"""Add FORK_OF relationships between existing GitHubRepository entities.
:param Neo4j neo4j:
Neo4j instance to add nodes to.
"""
query = '''
MATCH (fork:GitHubRepository), (parent:GitHubRepository)
WHERE fork.parentId = parent.id OR fork.sourceId = parent.id
CREATE (fork)-[:FORKS]->(parent)
'''
neo4j.run(query)
def add_app_data(packages: List[str], play_details_dir: str, neo4j: Neo4j):
"""Create nodes and relationships for Android apps.
:param List[str] packages:
List of package names to create :App and :GooglePlayPage nodes for.
:param str play_details_dir:
Name of directory to include JSON files from. Filenames in this
directory need to have .json extension. Filename without extension is
assumed to be package name for details contained in file.
:param Neo4j neo4j:
Neo4j instance to add nodes to.
"""
for package in packages:
__log__.info(
'Add :GooglePlayPage and :App nodes for package: %s', package)
add_google_play_page_node(package, neo4j, play_details_dir)
neo4j.run(
'''MERGE (g:GooglePlayPage {docId: {package}})
CREATE (a:App {id: {package}})-[:PUBLISHED_AT]->(g)''',
package=package)
def _main(args: argparse.Namespace):
"""Pass arguments to respective function."""
__log__.info('------- Arguments: -------')
__log__.info('PLAY_STORE_DETAILS_DIR: %s', args.PLAY_STORE_DETAILS_DIR)
__log__.info('REPO_DETAILS_DIR: %s', args.REPO_DETAILS_DIR)
__log__.info('REPOSITORY_LIST: %s', args.REPOSITORY_LIST.name)
__log__.info('--neo4j-host: %s', args.neo4j_host)
__log__.info('--neo4j-port: %d', args.neo4j_port)
__log__.info('------- Arguments end -------')
neo4j_user = os.getenv('NEO4J_USER')
__log__.info('Use `%s` to login to Neo4j', neo4j_user)
neo4j_password = os.getenv('NEO4J_PASSWORD')
__log__.info('Read Neo4j password from environment')
with Neo4j(NEO4J_HOST, neo4j_user, neo4j_password, NEO4J_PORT) as neo4j:
add_repository_info(
args.REPOSITORY_LIST, args.PLAY_STORE_DETAILS_DIR, neo4j,
args.REPO_DETAILS_DIR)
def add_repository_node(
meta_data: dict, snapshots: List[dict], neo4j: Neo4j) -> Node:
"""Add a repository and link it to all apps imnplemented by it.
Does not do anything if packages_names is empty or no :App node exists
with a matching package name.
:param dict meta_data:
Meta data of Google Play Store page parses from JSON.
:param List[Dict[str, str]] snapshots:
List of snapshots data. Must be length 1.
:param Neo4j neo4j:
Neo4j instance to add nodes to.
:returns Node:
The node created for the repository.
"""
snapshot = snapshots[0] if snapshots else {}
repo_data = format_repository_data(meta_data, snapshot)
query = '''
CREATE (repo:GitHubRepository {repo_properties})
RETURN repo
'''
result = neo4j.run(query, repo_properties=repo_data)
return result.single()[0]
def add_google_play_page_node(
package_name: str, neo4j: Neo4j, play_details_dir: str) -> Node:
"""Create a node for an Google Play page.
Meta data of Google Play page is loaded from JSON file at
<play_details_dir>/<package_name>.json
:param str package_name:
Package name.
:param Neo4j neo4j:
Neo4j instance to add nodes to.
:param str play_details_dir:
Name of directory to include JSON files from. Filenames in this
directory need to have .json extension. Filename without extension is
assumed to be package name for details contained in file.
:return Node:
Node created for Google Play page if JSON file exists, otherwise None.
"""
google_play_info = parse_google_play_info(package_name, play_details_dir)
if not google_play_info:
__log__.warning('Cannot create GooglePlayPage node %s.', package_name)
return None
__log__.info('Create GooglePlayPage node for %s.', package_name)
return neo4j.create_node('GooglePlayPage', **google_play_info)
def add_commit_nodes(commits: List[dict], repo_node_id: int, neo4j: Neo4j):
"""Create nodes representing GIT commits of a repository.
Creates a node for each commit and links it with the repository identified
by repo_node_id.
Also creates relationships to author, committer and parent commits. Creates
each of these in turn unless they exist already.
:param List[Dict[str, str]] commits:
List of data of commits.
:param int repo_node_id:
ID of node the commits should be linked to.
:param Neo4j neo4j:
Neo4j instance to add nodes to.
"""
for commit in commits:
parameters = {
'repo_id': repo_node_id,
'commit': {
'id': commit.get('id'),
'short_id': commit.get('short_id'),
'title': commit.get('title'),
'message': commit.get('message'),
'additions': commit.get('additions'),
'deletions': commit.get('deletions'),
'total': commit.get('total'),
},
'author': {
'email': commit.get('author_email'),
'name': commit.get('author_name'),
},
'committer': {
'email': commit.get('committer_email'),
'name': commit.get('committer_name'),
},
'authored_date': commit.get('authored_date'),
'committed_date': commit.get('committed_date'),
}
neo4j.run(
'''
MATCH (repo:GitHubRepository) WHERE id(repo) = {repo_id}
MERGE (commit:Commit {id: {commit}.id})
ON CREATE SET commit = {commit}
ON MATCH SET commit += {commit}
MERGE (author:Contributor {email: {author}.email})
ON CREATE SET author = {author}
ON MATCH SET author += {author}
MERGE (committer:Contributor {email: {committer}.email})
ON CREATE SET committer = {committer}
ON MATCH SET committer += {committer}
CREATE
(commit)-[:BELONGS_TO]->(repo),
(author)-[:AUTHORS {timestamp: {authored_date}}]->(commit),
(committer)-[:COMMITS {timestamp: {committed_date}}]->(commit)
''', **parameters)
for parent in commit.get('parent_ids').split(','):
neo4j.run(
'''
MATCH (c:Commit {id: {child}})
MERGE (p:Commit {id: {parent}})
CREATE (c)-[:PARENT]->(p)
''', parent=parent, child=commit.get('id'))
__log__.debug('Created commit %s', parameters['commit']['id'])