def create_queries(df_graphs, queue):
query_list = []
for g in df_graphs['files']:
print 'Folder:' + df_graphs['folder'], "File:" + g['file']
df_graph = DFGraph.deserialize(g['graph'])
queries = get_queries(df_graph)
if queries:
q_file = {'file': g['file'], 'queries': queries}
query_list.append(q_file)
if query_list:
query = {'folder': df_graphs['folder'], 'q_list': query_list}
queue.put(query)
def create_queries(df_graphs, queue):
query_list=[]
for g in df_graphs['files']:
print 'Folder:'+df_graphs['folder'],"File:"+g['file']
df_graph=DFGraph.deserialize(g['graph'])
queries=get_queries(df_graph)
if queries:
q_file={
'file':g['file'],
'queries':queries
}
query_list.append(q_file)
if query_list:
query={
'folder':df_graphs['folder'],
'q_list':query_list
}
queue.put(query)
def create_query(graph_text, Q_LIB):
lib = None
if Q_LIB == 'builtins':
lib = dir(__builtins__)
else:
lib = [Q_LIB]
try:
q_list = []
df_graph = json.loads(graph_text)
print "Foldername:" + df_graph['folder'], "File:" + df_graph['file']
graph = DFGraph.deserialize(df_graph['graph'])
for node in graph.dfs():
node_val = graph.graph_dict[node]
if isinstance(node_val, AssignmentNode):
for src in node_val.src:
dot_pos = src.find('.')
if dot_pos != -1:
if src[:dot_pos] in lib:
calls = graph.find_calls(node, src)
calls_list = []
if calls:
for call in calls:
call_dict = call.__dict__
map(call_dict.pop, [
'val', 'adjList', 'parent', 'op', 'src'
])
calls_list.append(call_dict)
if calls_list:
query = {
'folder': df_graph['folder'],
'file': df_graph['file'],
'type': src,
'obj': node_val.tgt,
'calls': calls_list,
'context': node_val.context
}
q_list.append(query)
break
except:
print "Error", sys.exc_info()
print 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno)
return q_list
def main():
manager = mp.Manager()
q = manager.Queue()
pool = mp.Pool(mp.cpu_count())
watcher = pool.apply_async(listener, (q, ))
jobs = []
graph_folder = 'graphs/'
for f_name in os.listdir(graph_folder):
graph = ""
with open(graph_folder + '/' + f_name, 'r') as file:
for line in file:
if line.strip() == '-' * 20:
try:
df_graphs = json.loads(graph)
for g in df_graphs['files']:
print 'Folder:' + df_graphs['folder'], "File:" + g[
'file']
df_graph = DFGraph.deserialize(g['graph'])
job = pool.apply_async(find_libs, (df_graph, q))
jobs.append(job)
graph = ""
except:
print "Unexpected error in worker:", sys.exc_info()[0]
with open("check-json.txt", 'w') as f:
f.write(graph)
break
else:
graph += line
for job in jobs:
try:
job.get()
except:
continue
q.put('kill')
pool.close()
pool.join()
def main():
manager=mp.Manager()
q=manager.Queue()
pool=mp.Pool(mp.cpu_count())
watcher = pool.apply_async(listener,(q,))
jobs=[]
graph_folder='graphs/'
for f_name in os.listdir(graph_folder):
graph=""
with open(graph_folder+'/'+f_name,'r') as file:
for line in file:
if line.strip()=='-' * 20:
try:
df_graphs=json.loads(graph)
for g in df_graphs['files']:
print 'Folder:'+df_graphs['folder'],"File:"+g['file']
df_graph=DFGraph.deserialize(g['graph'])
job=pool.apply_async(find_libs,(df_graph,q))
jobs.append(job)
graph=""
except:
print "Unexpected error in worker:", sys.exc_info()[0]
with open("check-json.txt",'w') as f:
f.write(graph)
break
else:
graph+=line
for job in jobs:
try:
job.get()
except:
continue
q.put('kill')
pool.close()
pool.join()
def create_query(graph_text, Q_LIB):
lib=None
if Q_LIB=='builtins':
lib=dir(__builtins__)
else:
lib=[Q_LIB]
try:
q_list=[]
df_graph=json.loads(graph_text)
print "Foldername:"+df_graph['folder'],"File:"+df_graph['file']
graph=DFGraph.deserialize(df_graph['graph'])
for node in graph.dfs():
node_val=graph.graph_dict[node]
if isinstance(node_val, AssignmentNode):
for src in node_val.src:
dot_pos=src.find('.')
if dot_pos!=-1:
if src[:dot_pos] in lib:
calls=graph.find_calls(node, src)
calls_list=[]
if calls:
for call in calls:
call_dict=call.__dict__
map(call_dict.pop, ['val','adjList','parent','op','src'])
calls_list.append(call_dict)
if calls_list:
query={
'folder':df_graph['folder'],
'file':df_graph['file'],
'type':src,
'obj':node_val.tgt,
'calls':calls_list,
'context':node_val.context
}
q_list.append(query)
break
except:
print "Error",sys.exc_info()
print 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno)
return q_list
def find_libs(graph_text, q):
graph = json.loads(graph_text)
folder, file = graph['folder'], graph['file']
print "Foldername:" + folder, "File:" + file
df_graph = DFGraph.deserialize(graph['graph'])
lib_count = Counter()
for node_num in df_graph.dfs():
node = df_graph.graph_dict[node_num]
if isinstance(node, AssignmentNode):
for src in node.src:
pos = src.find('.')
if pos != -1:
try:
lib = src[:pos]
if lib != 'self':
if lib in dir(__builtins__):
lib_count['builtins'] += 1
else:
lib_count[lib] += 1
except ImportError:
pass
q.put(lib_count)
return
def find_libs(graph_text, q):
graph=json.loads(graph_text)
folder, file =graph['folder'], graph['file']
print "Foldername:"+folder,"File:"+file
df_graph=DFGraph.deserialize(graph['graph'])
lib_count=Counter()
for node_num in df_graph.dfs():
node=df_graph.graph_dict[node_num]
if isinstance(node,AssignmentNode):
for src in node.src:
pos=src.find('.')
if pos!=-1:
try:
lib=src[:pos]
if lib!='self':
if lib in dir(__builtins__):
lib_count['builtins']+=1
else:
lib_count[lib]+=1
except ImportError:
pass
q.put(lib_count)
return