def get_project_information(project_name):
"""
:param project_name: the name of the current project.
:return: the query for general information about the current project.
"""
pypher_object = Pypher()
pypher_object.Match.node('u')
pypher_object.WHERE(__.u.__name__ == project_name)
pypher_object.RETURN('u')
return GenerateQuery.reformat_query(pypher_object)
def get_show_all_nodes_query(node_type=None):
"""
:param node_type: type of nodes to show.
:return: the query to show all nodes. Optional: Type of nodes to be recognized.
"""
# should be bundle, copilationUnit or package. Could be extracted via tracker.slots.get('nodeType').
pypher_object = Pypher()
if node_type is None:
pypher_object.Match.node('u')
else:
pypher_object.Match.node('u', labels=node_type)
pypher_object.RETURN('u')
return GenerateQuery.reformat_query(pypher_object)
def get_node_information_query(node_name, node_type=None):
"""
:param node_name: name of the node to get information about.
:return: the query for showing general information of one node (any entity) with a specific name.
"""
pypher_object = Pypher()
if node_type is None:
pypher_object.Match.node('u').where.u.__name__.CONTAINS(
Param('per_param', node_name))
else:
pypher_object.Match.node(
'u', labels=node_type).where.u.__name__.CONTAINS(
Param('per_param', node_name))
pypher_object.RETURN('u')
return GenerateQuery.reformat_query(pypher_object)
def get_largest_compilation_unit_query(bundle_name=None, order="mthd"):
"""
:param bundle_name: in entire project or in one specific bundle. Default is in all bundles.
:param order: largest by lines of code (loc) or number of methods (mthd). Default is "mthd".
:return: the query for the largest compilation unit.
'large' in this context means lines of code or number of methods.
It also can be the largest CU in a bundle or in general.
"""
pypher_object = Pypher()
if order == "loc":
pypher_object.Match.node('bundle', labels='bundles')\
.relationship('pkg', labels="Pkg_fragment").node('k')\
.relationship('kl', labels='compiled_By').node('cmp')
if bundle_name:
pypher_object.WHERE(__.bundle.__name__ == bundle_name)
pypher_object.RETURN('cmp')
pypher_object.OrderBy(__.cmp.__Loc__)
return GenerateQuery.reformat_query(pypher_object)
def get_by_ids(self, entity, ids):
p = Pypher()
p.MATCH
if isinstance(entity, Relationship):
var = 'rel'
p.node().rel(var, labels=entity.labels).node()
else:
var = 'node'
p.node(var, labels=entity.labels)
id_params = []
for i in ids:
key = 'entity_id_{}'.format(i)
id_params.append(Param(key, i))
p.WHERE.COR(__.ID(var).IN(*id_params))
p.RETURN(var)
return str(p), p.bound_params
def get_detailed_bundle_info_query(bundle_name, aspect=None):
"""
:param bundle_name: retrieve specific information for this bundle.
:param aspect: should be 'bundles', 'imports', 'Exports', 'packages','components','compilationUnit','Methods'.
:return: the query for (general or specific) information about one bundle.
"""
if aspect is None:
return GenerateQuery.get_node_information_query(
bundle_name, 'bundles')
pypher_object = Pypher()
if aspect == 'compilationUnit':
pypher_object.Match.node('u', labels='bundles').relationship('f', labels="Pkg_fragment").node('n')\
.relationship('c', labels="compiled_By").node("m")
elif aspect == 'methods':
pypher_object.Match.node('u', labels='bundles').relationship('pkg', labels="Pkg_fragment").node('k')\
.relationship('kl', labels='compiled_By').node('n')\
.relationship('r', labels='compiledUnits_topLevelType').node('nl')\
.relationship('rl', labels='Methods_Contains').node('m')
else:
if aspect == 'packages':
labels = 'uses_pkgs'
elif aspect == 'components':
labels = 'uses_components'
else:
labels = aspect
pypher_object.Match.node('u', labels='bundles').relationship(
'r', labels=labels).node('m')
pypher_object.Match.node('u').where.u.__name__.CONTAINS(
Param('per_param', bundle_name))
# this can be changed according to req. if we need all info or just names of packages
# query = str(self.pypherObject.RETURN('u.name', 'm.name'))
pypher_object.RETURN('u.name', 'm.name')
return GenerateQuery.reformat_query(pypher_object)
def get_by_search_string(self,
search_string,
ensure_privacy=True,
limit=options.pagination,
skip=0):
"""based on a search string of:
pyhon #code @mark
build a query with the foundation of:
MATCH (tag_resource:`Resource`)-[:`HasTag`]->(tag:`Tag`)
WHERE tag.tag_normalized = 'code'
WITH tag_resource
MATCH (user_resource:`Resource`)<-[:`AddedResource`]-(user:`User`)
WHERE id(user_resource) = id(tag_resource)
WITH user_resource, user
MATCH (resource:`Resource`)-[:`HasTag`]->(tag:`Tag`)
id(resource) = id(user_resource)
RETURN resource, user, collect(tag)
"""
from datcode.common.model.graph.node import User
search = parse_search_string(search_string)
user_mapper = self.get_mapper(User)
user_mapper.ensure_privacy = ensure_privacy
p = Pypher()
p.node('resource', labels='Resource')
p.rel_in(labels='AddedResource').node('user', labels='User')
p2 = Pypher()
p2.node('resource').rel_out(labels='HasTag').node('tag', labels='Tag')
p2 = Pypher()
p2.node('resource').rel_out(labels='HasTag').node('tag', labels='Tag')
p3 = Pypher()
p3.node('resource').rel_out(labels='HasTag').node('tags', labels='Tag')
query = Pypher()
query.MATCH(p, p2, p3)
wheres = []
search_ors = []
user_ors = []
tag_ors = []
# filter the resource title and descripiton by search string powersets
for contains in search['search']['contains']:
term = "(?i).*{}.*".format(contains)
d = Pypher()
t = Pypher()
d.resource.__description__.re(term)
t.resource.__title__.re(term)
search_ors.append(d)
search_ors.append(t)
if search_ors:
ors = Pypher()
ors.COR(*search_ors)
wheres.append(ors)
# filter by users
for user in search['users']:
u = Pypher()
u.user.__username__ == user
user_ors.append(u)
if user_ors:
ors = Pypher()
ors.COR(*user_ors)
wheres.append(ors)
# filter by tags
for tag in search['tags']:
u = Pypher()
u.tag.__tag_normalized__ == normalize(tag)
tag_ors.append(u)
if tag_ors:
ors = Pypher()
ors.COR(*tag_ors)
wheres.append(ors)
if wheres:
query.WHERE.CAND(*wheres)
# paginate and get a total count
total = query.clone()
total.RETURN('COUNT(DISTINCT(resource)) AS total')
total_res = self.mapper.query(pypher=total)
try:
total_results = total_res.first()['result']
except:
total_results = 0
query.RETURN('DISTINCT(resource)', 'user')
query.ORDERBY('resource.date_created').DESC
query.SKIP(skip).LIMIT(limit)
results = self.mapper.query(pypher=query)
result_data = []
all_tags = []
all_tag_ids = []
for res in results:
tags = self(res['resource'])['Tags']()
username = res['user']['username']
if ensure_privacy:
username = '******'
result_data.append({
'resource': self.data(res['resource']),
'user': {
'username': username,
},
'tags': self.data(tags),
})
for tag in tags:
if tag.id not in all_tag_ids:
all_tags.append(tag.data)
all_tag_ids.append(tag.id)
return {
'total': total_results,
'results': result_data,
'all_tags': all_tags,
}
class GenerateQuery:
def __init__(self, tracker):
self.extracted_intents = dict()
self.extracted_values = dict()
self.pypher_object = Pypher()
self.tracker = tracker
def get_simple_query(self):
extracted_intent = self.tracker.latest_message['intent']['name']
if len(self.tracker.latest_message['entities']) > 0:
extracted_value = self.tracker.latest_message['entities'][0][
'value']
extracted_entities = {
self.tracker.latest_message['entities'][index]['entity']:
self.tracker.latest_message['entities'][index]['value']
for index in range(len(
self.tracker.latest_message['entities']))
}
else:
extracted_entities = self.tracker.slots
if len(extracted_entities) <= 0:
return []
if extracted_intent == 'showNodeInformation':
if extracted_entities['node'] is None:
return []
self.pypher_object.Match.node('u').where.u.__name__.CONTAINS(
Param('per_param', extracted_entities['node']))
self.pypher_object.RETURN('u')
elif extracted_intent == 'showAllNodes':
self.pypher_object.Match.node(
'u',
labels=self.get_key_with_none_empty_value(extracted_entities))
self.pypher_object.RETURN('u')
elif extracted_intent == 'countAllNodes':
self.pypher_object.Match.node(
'u',
labels=self.get_key_with_none_empty_value(extracted_entities))
self.pypher_object.RETURN(__.count('u'))
elif extracted_intent == 'showLargestCompilationUnit':
if self.tracker.get_slot('Methods') is not None:
self.pypher_object.Match.node('bundle', labels='bundles').relationship \
('pkg', labels="Pkg_fragment").node('k').relationship \
('kl', labels='compiled_By').node().relationship \
('cp', labels="compiledUnits_topLevelType").node('n').relationship \
('rl', 'Methods_Contains').node('mthd')
self.pypher_object.RETURN('bundle.name', 'n.name',
__.count('mthd'))
self.pypher_object.OrderBy(__.count('mthd'))
else:
self.pypher_object.Match.node('bundle', labels='bundles').relationship \
('pkg', labels="Pkg_fragment").node('k').relationship \
('kl', labels='compiled_By').node('cmp')
if self.tracker.get_slot('bundles') is not None:
self.pypher_object.WHERE(
__.bundle.__name__ == self.tracker.get_slot('bundles'))
self.pypher_object.RETURN('bundle.name', 'cmp.name', 'cmp.Loc')
self.pypher_object.OrderBy(__.cmp.__Loc__)
self.pypher_object.Desc()
self.pypher_object.Limit(1)
elif extracted_intent == 'showDetailInfoBundles':
bundle_name = None
key_value = None
# iterate through all entities
for key, value in self.tracker.slots.items():
# key_value is assigned according to relation names
if key == 'bundles':
bundle_name = value
elif key == 'imports' or key == 'Exports':
key_value = value
elif key == 'packages':
key_value = 'uses_pkgs'
elif key == 'components':
key_value = 'uses_components'
elif key == 'compilationUnit':
key_value = 'compiled_By'
elif key == 'Methods':
key_value = 'Methods_Contains'
# this is relation name
if key_value == 'compiled_By':
self.pypher_object.Match.node('u', labels='bundles').relationship \
('f', labels="Pkg_fragment").node('n').relationship \
('c', labels="compiled_By").node("m")
elif key_value == 'Methods_Contains':
self.pypher_object.Match.node('u', labels='bundles').relationship \
('pkg', labels="Pkg_fragment").node('k').relationship \
('kl', labels='compiled_By').node('n').relationship \
('r', labels='compiledUnits_topLevelType').node('nl').relationship \
('rl', labels='Methods_Contains').node('m')
else:
self.pypher_object.Match.node('u', labels='bundles').relationship \
('r', labels=key_value).node('m')
self.pypher_object.WHERE(__.u.__name__ == bundle_name)
# this can be changed according to req. if we need all info or just names of packages
# query = str(self.pypherObject.RETURN('u.name', 'm.name'))
self.pypher_object.RETURN('u.name', 'm.name')
elif self.extracted_intents == 'showProjectInformation':
self.pypher_object.Match.node('u')
self.pypher_object.WHERE(
__.u.__name__ == self.tracker.get_slot('bundles'))
self.pypher_object.RETURN('u')
else:
if extracted_value is not None:
self.pypher_object.Match.node(
'u', labels=extracted_entities).WHERE.u.property(
'name') == extracted_value
self.pypher_object.RETURN('u')
query = str(self.pypher_object)
params = self.pypher_object.bound_params
return [query, params, extracted_intent]
@staticmethod
def get_key_with_none_empty_value(entities_dict):
for key, value in entities_dict.items():
print('key: ' + key)
if value:
print('value: ' + value)
if key != "project":
return key
return {}
def convert_text_to_query(self):
error = None
[query, params, extracted_intent] = self.get_simple_query()
if query is None or params is None or extracted_intent is None:
error = "no Query written regarding this intention or intent prediction is not valid"
return [query, params, self.tracker.latest_message['intent'], error]
class RelatedEntityQuery(_BaseQuery):
def __init__(self,
direction='out',
relationship_entity=None,
relationship_type=None,
relationship_properties=None,
start_entity=None,
end_entity=None,
params=None,
single_relationship=False,
start_query_variable='start_node',
relationship_query_variable='relt',
end_query_variable='end_node'):
super(RelatedEntityQuery, self).__init__()
self.direction = direction
self.relationship_entity = relationship_entity
self.relationship_type = relationship_type
self.start_query_variable = start_query_variable
self.relationship_query_variable = relationship_query_variable
self.end_query_variable = end_query_variable
self._start_entity = None
self.start_entity = start_entity
self._end_entity = None
self.end_entity = end_entity
self.relationship_properties = relationship_properties or {}
self.params = params
self.skip = None
self.limit = 1 if single_relationship else None
def reset(self):
super(RelatedEntityQuery, self).reset()
self.skip = None
self.limit = None
if self.start_entity:
self.start_entity.query_variable = self.start_query_variable
VM.set_query_var(self.start_entity)
if self.end_entity:
self.end_entity.query_variable = self.end_query_variable
VM.set_query_var(self.end_entity)
def _get_relationship_entity(self):
return self._relationship_entity
def _set_relationship_entity(self, relationship):
if relationship is not None and not isinstance(relationship,
Relationship):
raise AttributeError('Must be an <Relationship> and not'
' a <{t}>'.format(t=type(relationship)))
self._relationship_entity = relationship
return self
relationship_entity = property(_get_relationship_entity,
_set_relationship_entity)
def _get_start_entity(self):
return self._start_entity
def _set_start_entity(self, entity):
if entity is not None and not isinstance(entity, Node):
raise AttributeError('entity must be a Node instance')
if entity:
entity.query_variable = self.start_query_variable
VM.set_query_var(entity)
self._start_entity = entity
start_entity = property(_get_start_entity, _set_start_entity)
def _get_end_entity(self):
return self._end_entity
def _set_end_entity(self, entity):
if entity is not None and not isinstance(entity, Node):
raise AttributeError('entity must be a Node instance')
if entity:
entity.query_variable = self.end_query_variable
VM.set_query_var(entity)
self._end_entity = entity
end_entity = property(_get_end_entity, _set_end_entity)
def _build_start(self):
pypher = Pypher()
if self.start_entity.id is not None:
qv = self.start_entity.query_variable
where = __.ID(qv) == self.start_entity.id
pypher.NODE(qv)
self.wheres.append(where)
else:
pypher.NODE(self.start_query_variable,
labels=self.start_entity.labels)
return pypher
def _build_end(self):
pypher = Pypher()
qv = self.end_query_variable
labels = None
if self.end_entity:
qv = self.end_entity.query_variable
labels = self.end_entity.labels
pypher.NODE(qv, labels=labels)
self.returns.append(qv)
return pypher
def _build_relationship(self):
pypher = Pypher()
if self.relationship_entity:
self.relationship_entity.query_variable =\
self.relationship_query_variable
VM.set_query_var(self.relationship_entity)
pypher.relationship(self.relationship_entity.query_variable,
direction=self.direction,
labels=self.relationship_entity.labels,
**self.relationship_properties)
else:
pypher.relationship(direction=self.direction,
labels=self.relationship_type)
return pypher
def query(self, return_relationship=False, returns=None):
if not self.start_entity:
raise RelatedQueryException(('Related objects must have a'
' start entity'))
self.pypher = Pypher()
pypher = self.pypher
self.matches.insert(0, self._build_end())
self.matches.insert(0, self._build_relationship())
self.matches.insert(0, self._build_start())
self.pypher.MATCH
for match in self.matches:
self.pypher.append(match)
if self.wheres:
self.pypher.WHERE.CAND(*self.wheres)
if return_relationship:
ret = getattr(__, self.relationship_query_variable)
self.returns = [
ret,
]
returns = returns or self.returns
self.pypher.RETURN(*returns)
if self.orders:
self.pypher.ORDER.BY(*self.orders)
if self.skip is not None:
self.pypher.SKIP(self.skip)
if self.limit is not None:
self.pypher.LIMIT(self.limit)
self.reset()
return str(pypher), pypher.bound_params
def connect(self, entity, properties=None):
if not self.start_entity:
message = ('The relationship {} does not have a start'
' entity'.format(self.relationship_entity
or self.relationship_type))
raise RelatedQueryException(('The relationship {} does not '))
if self.start_entity == entity:
msg = ('the start entity {} cannot be the same as the'
' end entity'.format(entity))
raise RelatedQueryException(msg)
start = self.start_entity
end = entity
if self.direction == 'in':
start, end = end, start
kwargs = {
'start': start,
'end': end,
'properties': properties,
}
if self.relationship_entity:
return self.relationship_entity.__class__(**kwargs)
kwargs['labels'] = self.relationship_type
return Relationship(**kwargs)
def delete(self, entity):
if isinstance(entity, Relationship):
if entity.id is not None:
query = Query(entity)
return query.delete()
elif entity.end and hasattr(entity.end, 'id'):
self.matches.insert(0, self._build_end())
self.matches.insert(0, self._build_relationship())
self.matches.insert(0, self._build_start())
self.pypher.MATCH
for match in self.matches:
self.pypher.append(match)
self.pypher.DETACH.DELETE(self.relationship_query_variable)
self.reset()
return str(self.pypher), self.pypher.bound_params
def delete_by_entity_id(self, *ids):
if not ids:
msg = 'There must be ids passed in to the delete method'
raise AttributeError(msg)
def _build_start():
pypher = Pypher()
if self.start_entity.id is not None:
qv = self.start_entity.query_variable
where = __.ID(qv) == self.start_entity.id
pypher.NODE(qv)
self.wheres.append(where)
else:
pypher.NODE(self.start_query_variable)
return pypher
self.pypher = Pypher()
self.matches.insert(0, self._build_end())
self.matches.insert(0, self._build_relationship())
self.matches.insert(0, _build_start())
self.pypher.MATCH
for match in self.matches:
self.pypher.append(match)
id_params = []
for i in ids:
key = 'end_id_{}'.format(i)
id_params.append(Param(key, i))
self.wheres.append(__.ID(self.end_query_variable).IN(*id_params))
_id = __.ID(self.start_query_variable)
if self.start_entity.id is not None:
self.wheres.append(_id == self.start_entity.id)
# else:
# wheres.append(_id)
self.pypher.WHERE.CAND(*self.wheres)
self.pypher.DELETE(self.relationship_query_variable)
self.reset()
return str(self.pypher), self.pypher.bound_params
