class EvidenceRpcClient(object):
def __init__(self, coupled=False):
self.tree_importer = JsonImporter()
self.connection = pika.BlockingConnection(
pika.ConnectionParameters(host='localhost'))
self.channel = self.connection.channel()
self.coupled = coupled
result = self.channel.queue_declare(exclusive=True)
self.callback_queue = result.method.queue
self.channel.basic_consume(self.on_response,
no_ack=True,
queue=self.callback_queue)
def deserialize(self, response):
res = json.loads(response)
if self.coupled:
res['tree'] = self.tree_importer.import_(res['tree'])
else:
res[0] = json.loads(res[0])
res[0]['tree'] = self.tree_importer.import_(res[0]['tree'])
return res
def on_response(self, ch, method, props, body):
if self.corr_id == props.correlation_id:
self.response = body
def call(self, request):
self.response = None
self.corr_id = str(uuid.uuid4())
try:
self.channel.basic_publish(exchange='',
routing_key='rpc_queue',
properties=pika.BasicProperties(
reply_to=self.callback_queue,
correlation_id=self.corr_id,
),
body=str(request))
except:
time.sleep(5)
self.channel.basic_publish(exchange='',
routing_key='rpc_queue',
properties=pika.BasicProperties(
reply_to=self.callback_queue,
correlation_id=self.corr_id,
),
body=str(request))
while self.response is None:
self.connection.process_data_events()
return self.deserialize(self.response)
def get_rank_kcr(dimacs_, solset_, jsonfile_):
# read tree structure from file
_treefile = os.path.dirname(dimacs_) + '/smarch/tree.json'
# node = AnyNode(count=-1, cube=[])
if os.path.exists(jsonfile_):
with open(jsonfile_, 'r') as file:
data = file.read()
importer = JsonImporter()
_root = importer.import_(data)
total = _root.count
else:
print("ERROR: tree file not found!")
return
for sol in solset_:
# traverse tree based on solution
_node = _root
_precision = 0
_number = 0
while _precision == 0:
_node, _precision, _number = traverse_cube(_node, sol, _number)
if _precision > 0:
print(str(_number / total) + "," + str(_precision / total))
else:
print("ERROR: tree traverse failure")
def _loadTree(self, filename, erease):
str_tree = None
index = 0
if filename is None or not os.path.exists(filename):
return None
if erease:
with open(filename, 'w') as file:
file.write("")
with open(filename, 'r') as file:
try:
index = int(file.readline())
except ValueError:
return None
str_tree = file.read()
importer = JsonImporter()
try:
tree = importer.import_(str_tree)
except json.JSONDecodeError:
return None
return index, tree
def _restore_json(self, string):
'''restore the tree from json'''
imp = JsonImporter()
root = imp.import_(string)
if self.verbose:
Logger.info('Catalog imported from json \"{}\"'.format(self.path))
return root
def ast_json_to_anytree(json_data):
"""
Deserializes an anytree in json format and returns its root node.
"""
from anytree.importer import JsonImporter
importer = JsonImporter()
return importer.import_(json_data)
def get_rank(dimacs_, dir_, jsonfile_):
# read dimacs file for feature list
_features, _clauses, _vars = read_dimacs(dimacs_)
if os.path.exists(jsonfile_):
with open(jsonfile_, 'r') as file:
data = file.read()
importer = JsonImporter()
_root = importer.import_(data)
total = _root.count
else:
print("ERROR: tree file not found!")
return
_cdir = dir_
for file in os.listdir(_cdir):
if file.endswith('.config'):
# convert config file into variable list
sol = read_config_kmax(_features, _cdir + "/" + file)
# traverse tree based on solution
_node = _root
_precision = 0
_number = 0
while _precision == 0:
_node, _precision, _number = traverse_cube(_node, sol, _number)
if _precision > 0:
print(str(_number / total) + "," + str(_precision / total))
else:
print("ERROR: tree traverse failure")
def __init__(self, vectorifier=None):
dict_importer = DictImporter(nodecls=Node)
importer = JsonImporter(dictimporter=dict_importer)
self.root = importer.import_(symptom_json)
if vectorifier is not None:
# normal mode, if it is None is eval mode
self.vectorifier = vectorifier
self.vector_dimension = vectorifier.d
"""
if concept name vectors file not exists, create it
"""
if vectorifier.internal_representation == "glove":
if not os.path.isfile(COM.CSV_ST_CONCEPT_NAMES_GLOVE_PATH + \
str(self.vector_dimension) + "d.csv"):
print(
"Concept names glove vectors file not found. \nComputing file..."
)
self.__save_csv_concept_name_glove_vectors()
self.concept_name_vectors_df = pd.read_csv(COM.CSV_ST_CONCEPT_NAMES_GLOVE_PATH + \
str(self.vector_dimension) + "d.csv",
header=None)
elif vectorifier.internal_representation == "bert":
if not os.path.isfile(COM.CSV_ST_CONCEPT_NAMES_BERT_PATH + \
COM.FILENAME_CSV_ST_CONCEPT_NAMES_BERT):
print(
"Concept names bert vectors file not found. \nComputing file..."
)
self.__save_csv_concept_name_bert_vectors()
self.concept_name_vectors_df = pd.read_csv(COM.CSV_ST_CONCEPT_NAMES_BERT_PATH + \
COM.FILENAME_CSV_ST_CONCEPT_NAMES_BERT,
header=None)
def get_tree(self, file_name):
''' This function is used to import json file and return anytree Node '''
importer = JsonImporter()
tree_file = open(file_name, "r")
data = tree_file.read()
tree_file.close()
return importer.import_(data)
def read_tree(file_path: str):
"""Deserialize a tree from a saved JSON file"""
importer = JsonImporter()
with open(file_path, 'r') as j:
json_tree = j.read().replace("_name", "name")
root = importer.import_(json_tree)
root.cost = None
return root
def _deserialize_ontologies(filename):
"""Deserializes an ontology from a JSON file and returns its root."""
importer = JsonImporter()
forest = []
with open(filename, 'r') as f:
tree_list = json.load(f)
for tree_text in tree_list:
forest.append(importer.import_(tree_text))
return forest
def read(self) -> Entry:
content = None
if os.path.isfile(self._filepath):
fp = open(Path(self._filepath))
d_imp = DictImporter(nodecls=Entry)
importer = JsonImporter(dictimporter=d_imp)
content = importer.import_(fp.read())
fp.close()
return content
def draw_tree(string1, path_des, file):
from anytree import RenderTree
importer = JsonImporter()
root = importer.import_(string1)
# Render Tree
print(RenderTree(root, style=ContRoundStyle()))
# # Render graph tree
DotExporter(root).to_picture(path_des + file)
Image(filename=path_des + file)
def load(self):
extension = ".rpt"
dir_name = "../data/policies/"
file_name = dir_name + self.agent_name + extension
importer = JsonImporter()
data = None
with open(file_name, "r") as f:
data = json.loads(f)
f.close()
self.tree = importer.import_(data)
def load_tree(self):
with open('test.json') as json_file:
data = json.load(json_file)
print(data)
print("\n")
data_json = json.dumps(data)
importer = JsonImporter()
d0 = importer.import_(data_json)
print(RenderTree(d0))
def import_conversation_trees_from_db(user_id):
documents = collection_trees.find()
trees = []
for i, tree in enumerate(documents):
importer = JsonImporter()
r1 = json_util.dumps(tree)
root = importer.import_(r1)
if i % 1000 == 0:
print("Finding: {}".format(i))
trees.append(root)
return trees
def __init__(self, dir='controllers/'):
importer = JsonImporter()
with open(dir + 'data.json', 'r') as f:
data = json.load(f)
self.root = importer.import_(data)
self.current_node = self.root
DotExporter(self.root).to_picture("tree.png")
def printTree(tweetsFile):
propTree = PropTree() # an instance of a tree
importer = JsonImporter()
rootNr = 0
with open(tweetsFile, 'r') as _file:
content = _file.read()
contentSplit = content.split("&")
contentSplit.pop()
for data in contentSplit:
root = importer.import_(data)
propTree.addRoot(root)
rootNr += 1
return propTree
def drawtree(string, path_des, path, filename):
try:
error_flag = 0
importer = JsonImporter()
root = importer.import_(string)
print(RenderTree(root, style=ContRoundStyle()))
DotExporter(root).to_picture(path_des + filename)
Image(filename=path_des + filename)
return (error_flag)
except:
error_flag = 1
f = open(path + '/H_log.dat', 'a+')
f.write(filename + '\tInvalid JSON format\n')
f.close()
return (error_flag)
def printTree(tweetsFile):
propTree = PropTree() # an instance of a tree
importer = JsonImporter()
rootNr = 0
with open('./data/tree/trees/' + tweetsFile + '.txt', 'r') as _file:
content = _file.read()
contentSplit = content.split("&")
for data in contentSplit:
root = importer.import_(data)
propTree.addRoot(root)
rootNr += 1
propTree.makeSimpleTree()
return propTree
def drawtree(string, path_des, path, filename, file):
try:
error_flag = 0
importer = JsonImporter()
root = importer.import_(string)
file1 = file+'.dot'
print(RenderTree(root, style=ContRoundStyle()))
DotExporter(root).to_dotfile(path_des+file1)
add_edge_labels(path_des, file1)
check_call(['dot','-Tpng',path_des+file1,'-o',path_des+file+'.png'])
return(error_flag)
except:
error_flag = 1
f = open(path+'/E_log.dat', 'a+')
f.write(filename +'\tInvalid Drawtree input format\n')
f.close()
f = open(path_des+'/E_log.dat', 'a+')
f.write('\tInvalid Drawtree input format\n')
f.close()
return(error_flag)
def drawtree(string, path_des, filename): importer = JsonImporter() root = importer.import_(string) print(RenderTree(root, style=ContRoundStyle())) DotExporter(root).to_picture(path_des+filename) Image(filename=path_des+filename)
def JSON_toTree(json_data):
importer = JsonImporter()
ITSM_tree = importer.import_(json_data)
print(RenderTree(ITSM_tree))
return ITSM_tree
def get_rank_unigen(dimacs_, sampleFile_):
# read dimacs file for feature list
_features, _clauses, _vars = read_dimacs(dimacs_)
# read tree structure from file
_treefile = os.path.dirname(dimacs_) + '/smarch/tree.json'
# node = AnyNode(count=-1, cube=[])
if os.path.exists(_treefile):
with open(_treefile, 'r') as file:
data = file.read()
importer = JsonImporter()
_root = importer.import_(data)
total = _root.count
else:
print("ERROR: tree file not found!")
return
with open(sampleFile_, 'r') as f:
i = 1
for line in f:
line = line[1:len(line)]
raw = line.split()
if len(raw) != 0:
sol = raw[1:len(raw) - 1]
#print(sol)
# traverse tree based on solution
_node = _root
_precision = 0
_number = 0
_cdir = os.path.dirname(dimacs_) + '/'
sdimacs = _cdir + "/sample.dimacs"
ssol = _cdir + "/" + "sample.sol"
gen_dimacs(_vars, _clauses, sol, sdimacs)
getoutput("minisat " + sdimacs + " " + ssol)
_outfile = os.path.dirname(sampleFile_) + '/validCheck.dimacs'
gen_dimacs(_vars, _clauses, sol, _outfile)
res = getoutput(SHARPSAT + ' -q ' + _outfile)
fsol = list()
with open(ssol, 'r') as f:
fsol = f.read().split()
del fsol[0]
if is_int(res):
print(res, end=',')
while _precision == 0:
_node, _precision, _number = traverse_cube(
_node, fsol, _number)
if _precision > 0:
print(
str(_number / total) + "," +
str(_precision / total))
else:
print("ERROR: tree traverse failure")
else:
print("ERROR: sample invalid")
i += 1
for pre, fill, node in RenderTree(root):
fileTreeLog.write("%s%s\n" % (pre, node.name))
fileTreeLog.close()
if mode == 'pred' or mode == 'kValid':
sequences = sequencesVal
ys = classesVal
#import tree from json file
if mode == 'pred':
with open(treeSource) as infile:
jsonTree = json.load(infile)
importer = JsonImporter()
root = importer.import_(jsonTree)
#przeprowadz predykcje na zbiorze
y_preds = predictBatch(sequences, root)
y_preds = [
y_pred[0] if len(y_pred) == 1 else int(np.median(y_pred))
for y_pred in y_preds
]
answers = {'good': 0, 'bad': 0}
for y, y_pred in zip(ys, y_preds):
if y == y_pred:
answers['good'] += 1
else:
answers['bad'] += 1
print("Wynik predykcji: " +
str(answers['good'] / (answers['good'] + answers['bad'])))
class AstToCodeParser:
def __init__(self, output_folder, csv_file_path='', use_compression=False, processes_num=1, tokenized=True):
# CSV to get program data from
self.csv_file_path = csv_file_path
# Output folder to save data to
self.output_folder = output_folder
self.tokenized = tokenized
# Create JSON importer
self.importer = JsonImporter()
# Boolean indicating whether ASTs from the input folder are stored using compression
self.use_compression = use_compression
# Number of parallel processes
self.processes_num = processes_num
def load_vocabs_from_folder(self, folder):
if self.tokenized:
self.tokenizers = {}
for dirs, _, files in os.walk(folder):
for file in files:
if 'tokens.json' in file:
self.tokenizers[file.split('_')[0]] = Tokenizer(self.output_folder, self.tokenized)
self.tokenizers[file.split('_')[0]].load_from_path(os.path.join(dirs, file))
def load_vocabs_from_dicts(self, token2index_dicts):
if self.tokenized:
self.tokenizers = {}
for vocab_type, token2index in token2index_dicts.items():
self.tokenizers[vocab_type] = Tokenizer(self.output_folder, self.tokenized)
self.tokenizers[vocab_type].load_from_dict(token2index)
def get_label(self, node):
if self.tokenized:
if node.res:
return self.tokenizers['RES'].get_label(node.token)
else:
parent_label = self.tokenizers['RES'].get_label(node.parent.token)
if 'LITERAL' in parent_label:
return self.tokenizers['LITERAL'].get_label(node.token)
elif 'TYPE' == parent_label:
return self.tokenizers['TYPE'].get_label(node.token)
elif 'REF_BUILTIN' == parent_label:
return self.tokenizers['NAME_BUILTIN'].get_label(node.token)
else:
return self.tokenizers['NAME'].get_label(node.token)
else:
return node.token
def merge_terminals(self, terminals):
return ' '.join([self.get_label(term) for term in terminals])
def get_operator(self, ast_item):
operator = self.get_label(ast_item).split('_')[-1]
code = ''
if 'BINARY' in self.get_label(ast_item) or 'COMPOUND_ASSIGNMENT' in self.get_label(ast_item):
if len(ast_item.children) > 0:
code += self.parse_node(ast_item.children[0])
code += f' {operator} '
if len(ast_item.children) > 1:
code += self.parse_node(ast_item.children[1])
elif 'UNARY' in self.get_label(ast_item):
if 'POST' in self.get_label(ast_item):
if len(ast_item.children) > 0:
code += self.parse_node(ast_item.children[0])
code += operator
else:
code += operator + ' '
if len(ast_item.children) > 0:
code += self.parse_node(ast_item.children[0])
elif 'CONDITIONAL_OPERATOR' == self.get_label(ast_item):
if len(ast_item.children) > 0:
code += self.parse_node(ast_item.children[0])
code += ' ? '
if len(ast_item.children) > 1:
code += self.parse_node(ast_item.children[1])
code += ':'
if len(ast_item.children) > 2:
code += self.parse_node(ast_item.children[2])
else:
print(ast_item)
pass
return code
def get_call_exp_operator(self, ast_item):
code = ''
operator_label = self.get_label(ast_item.children[0].children[0])
operator = operator_label.split('operator')[-1]
if operator == '[]':
if len(ast_item.children[1].children) > 0:
code += self.parse_node(ast_item.children[1].children[0])
if len(ast_item.children[1].children) > 1:
code += f'[{self.parse_node(ast_item.children[1].children[1])}]'
elif operator == '()':
if len(ast_item.children[1].children) > 0:
code += self.parse_node(ast_item.children[1].children[0])
if len(ast_item.children[1].children) > 1:
code += f'({self.parse_node(ast_item.children[1].children[1])})'
else:
code += '()'
elif operator == '*' or (len(ast_item.children) > 1 and len(ast_item.children[1].children) < 2):
code += operator
if len(ast_item.children[1].children) > 0:
code += self.parse_node(ast_item.children[1].children[0])
elif operator in ['++_PRE', '++_POST', '--_PRE', '--_POST']:
if 'PRE' in operator:
code += operator.split('_')[0]
if len(ast_item.children[1].children) > 0:
code += self.parse_node(ast_item.children[1].children[0])
else:
if len(ast_item.children[1].children) > 0:
code += self.parse_node(ast_item.children[1].children[0])
code += operator.split('_')[0]
else:
# first argument of operator
if len(ast_item.children) > 1 and len(ast_item.children[1].children) > 0:
code += self.parse_node(ast_item.children[1].children[0])
code += f' {operator} '
if len(ast_item.children) > 1 and len(ast_item.children[1].children) > 1:
code += self.parse_node(ast_item.children[1].children[1])
return code
def get_var_decl(self, ast_item):
# Contains all non-ref and non-type values
# e.g. int ch[1][1] = {{2}} would give [1, 1, 2]
declarations = []
acc_spec = ''
var_type = ''
var_name = 'VAR_NAME_PLACEHOLDER'
# If the variable is an array, this is will be filled
array_sizes = []
for child in ast_item.children:
# Skip the type if this is a child of a declaration statement and not the first child
# and the type is the same as the type of the first child
if self.get_label(child) == 'TYPE_KIND' and \
not (self.get_label(ast_item.parent) == 'DECL_STMT' \
and ast_item != ast_item.parent.children[0]) and len(child.children) > 0:
var_type_size = self.get_type(child.children[0])
if type(var_type_size) is tuple:
var_type += var_type_size[0]
array_sizes = var_type_size[1]
else:
var_type = var_type_size
elif self.get_label(child) == 'TYPE_KIND':
var_type_size = self.get_type(child.children[0])
if type(var_type_size) is tuple:
array_sizes = var_type_size[1]
elif self.get_label(child) == 'DECLARATOR':
for decl_child in child.children:
if self.get_label(decl_child) == 'NAME':
var_name = self.merge_terminals(decl_child.children)
else:
declarations.append(self.parse_node(decl_child))
elif self.get_label(child) == 'ACCESS_SPECIFIER':
acc_spec += f'{self.merge_terminals(child.children).lower()}:\n'
ref_dims = ['[' + num + ']' for num in array_sizes]
# initialize value the variable was initialized to (empty)
var_value = ''
# append initial values if set
if len(declarations) > 0:
var_value += ' = ' + ' '.join(declarations)
# Combine the type of the variable with the name, dimensions and initial value
return f'{acc_spec}{var_type} {var_name}{"".join(ref_dims)}{var_value}'
def get_type(self, ast_item):
type_string = ''
if self.get_label(ast_item) == 'POINTER':
type_string += self.get_type(ast_item.children[0])
type_string += ' *'
return type_string
elif self.get_label(ast_item) == 'CONST_QUALIFIED':
type_string += 'const '
type_string += self.get_type(ast_item.children[0])
return type_string
elif self.get_label(ast_item) in ['TYPE', 'TYPE_REF']:
if len(ast_item.children) > 0:
return self.get_label(ast_item.children[0])
else:
return ''
elif self.get_label(ast_item) == 'TYPE_ARRAY':
array_sizes = []
array_type = 'auto'
for array_child in ast_item.children:
if self.get_label(array_child) in ['TYPE', 'TYPE_REF', 'TYPE_RECORD']:
array_type = self.get_type(array_child)
elif self.get_label(array_child) == 'ARRAY_SIZES':
for array_size in array_child.children:
array_sizes.append(self.parse_node(array_size))
type_string += array_type
return (type_string, array_sizes)
elif self.get_label(ast_item) == 'TYPE_RECORD':
record_type = 'auto'
for record_child in ast_item.children:
if self.get_label(record_child) in ['TYPE', 'TYPE_REF']:
record_type = self.get_type(record_child)
type_string += record_type
for record_child in ast_item.children:
if self.get_label(record_child) == 'TYPE_RECORD_ELEMENTS':
type_string += '<'
for index, record_element in enumerate(record_child.children):
type_string += self.get_type(record_element)
if index < len(record_child.children)- 1:
type_string += ', '
type_string += '>'
return type_string
elif self.get_label(ast_item) == 'LVALUEREFERENCE':
return self.get_type(ast_item.children[0]) + '&'
elif self.get_label(ast_item) == 'RVALUEREFERENCE':
return self.get_type(ast_item.children[0]) + '&&'
else:
return self.get_label(ast_item)
def get_function_decl(self, ast_item):
params = []
acc_spec = ''
return_type = ''
const = ''
func_name = ''
for child in ast_item.children:
if self.get_label(child) == 'TYPE_KIND':
return_type += self.get_type(child.children[0])
elif self.get_label(child) == 'PARM_DECL':
params.append(self.get_var_decl(child))
elif self.get_label(child) == 'NAME':
func_name = self.merge_terminals(child.children)
elif self.get_label(child) == 'ACCESS_SPECIFIER':
acc_spec += f'{self.get_label(child.children[0]).lower()}:\n'
elif self.get_label(child) == 'CONST':
const += ' const'
return f'{acc_spec}{return_type} {func_name}({", ".join(params)}){const}'
def is_function(self, node):
return self.get_label(node) in ['FUNCTION_DECL', 'FUNCTION_TEMPLATE',
'CXX_METHOD', 'DESTRUCTOR', 'CONSTRUCTOR']
def get_temp_type_param(self, ast_item):
parameter_pack = False
# If it is the last template type parameter, check if we need to add ... (=parameter pack -> zero or more params)
if ast_item == ast_item.parent.children[-1]:
function_decl_index = -1
for index, child in enumerate(ast_item.parent.parent.children):
if child == ast_item.parent:
function_decl_index = index + 1
if len(ast_item.parent.children) > function_decl_index and len(ast_item.parent.children[function_decl_index].children) > 1:
last_parm_decl = ast_item.parent.parent.children[function_decl_index].children[-2].children[0].children[0]
if self.get_label(last_parm_decl).endswith('...'):
parameter_pack = True
out = f'typename{"..." if parameter_pack else ""}'
if len(ast_item.children) > 0:
return out + f'{self.get_label(ast_item.children[0])}'
else:
return out
def get_lambda_expr(self, ast_item):
capture_clauses = []
params = []
for child in ast_item.children:
if self.get_label(child) == 'CAPTURE_CLAUSE':
capture_clauses.append(self.merge_terminals(child.children))
elif self.get_label(child) == 'PARM_DECL':
params.append(self.get_var_decl(child))
return f'[{", ".join(capture_clauses)}]({", ".join(params)})'
def parse_node(self, node):
call_exp_operators = ['[]', '=', '<<', '>>', '==', '+', '-', '%', '*', '/',
'+=', '-=', '^=', '||', '()', '!=',
'++_PRE', '++_POST', '--_PRE', '--_POST']
call_exp_operator_labels = ['operator' + op for op in call_exp_operators]
code = ''
if self.get_label(node) == 'VAR_DECL' or self.get_label(node) == 'FIELD_DECL' or self.get_label(node) == 'UNEXPOSED_DECL':
code += self.get_var_decl(node)
elif self.is_function(node):
code += self.get_function_decl(node)
for child in node.children:
code += self.parse_node(child)
elif self.get_label(node) == 'COMPOUND_STMT':
code += ' {\n'
for child in node.children:
code += self.parse_node(child)
code += '}\n'
elif self.get_label(node) == 'PAREN_EXPR':
code += '('
for child in node.children:
code += self.parse_node(child)
code += ')'
elif 'OPERATOR' in self.get_label(node) or 'UNARY_EXPR' in self.get_label(node):
code += self.get_operator(node)
elif self.get_label(node) == 'GNU_NULL_EXPR':
code += 'NULL'
elif self.get_label(node) == 'CXX_NULL_PTR_LITERAL_EXPR':
code += 'nullptr'
elif self.get_label(node) == 'COMPOUND_LITERAL_EXPR' or self.get_label(node) == 'CSTYLE_CAST_EXPR':
code += '('
code += self.parse_node(node.children[0]) if self.get_label(node) == 'COMPOUND_LITERAL_EXPR' else self.get_type(node.children[0].children[0])
code += ')'
for child in node.children[1:]:
code += self.parse_node(child)
elif self.get_label(node) in ['DECL_REF_EXPR', 'MEMBER_REF_EXPR', 'MEMBER_REF', 'LABEL_REF', 'REF', 'REF_BUILTIN'] or 'LITERAL' in self.get_label(node):
for child in node.children[1:]:
code += self.parse_node(child)
code += '.'
if self.get_label(node) == 'MEMBER_REF_EXPR' and len(node.children) > 1 \
and self.get_label(node.children[1]) != 'CXX_THIS_EXPR': #node.parent.parent) == 'REF':
code += '.'
if len(node.children) > 0:
code += self.get_label(node.children[0])
# if self.get_label(node.parent) in ['REF', 'REF_BUILTIN']:
# code += '.'
elif self.get_label(node) == 'IF_STMT':
code += 'if('
# The first compound statement of an if statement is the code of the
# if statement, the second (and also last) compound statement is for
# a possible else statement
compound_statements = 0
for index, child in enumerate(node.children):
if index == len(node.children) - 1 and compound_statements > 0:
code += 'else '
if self.get_label(child) == 'COMPOUND_STMT':
compound_statements += 1
code += self.parse_node(child)
if index == 0:
code += ')'
elif self.get_label(node) == 'ARRAY_SUBSCRIPT_EXPR':
if len(node.children) > 0:
code += self.parse_node(node.children[0])
code += '['
if len(node.children) > 1:
code += self.parse_node(node.children[1])
code += ']'
elif self.get_label(node) == 'RETURN_STMT':
code += 'return'
for child in node.children:
code += ' '
code += self.parse_node(child)
elif self.get_label(node) == 'FOR_STMT':
code += 'for('
# Count amount of expressions in for loop (can essentialy be empty: for(;;))
for_stmt_expressions = 0
for child in node.children:
if self.get_label(child) != 'COMPOUND_STMT':
for_stmt_expressions += 1
else:
break
# add ; for empty expressions e.g. if we have 2 expressions: for(;expr2;expr3)
for _ in range(3 - for_stmt_expressions):
code += ';'
# add expressions
for i in range(for_stmt_expressions):
code += self.parse_node(node.children[i])
if i < for_stmt_expressions - 1:
code += '; '
code += ')'
# parse for loop compound statement
for i in range(for_stmt_expressions, len(node.children)):
code += self.parse_node(node.children[i])
elif self.get_label(node) == 'CALL_EXPR':
if len(node.children) > 0 and len(node.children[0].children) > 0 and self.get_label(node.children[0].children[0]) in call_exp_operator_labels:
code += self.get_call_exp_operator(node)
else:
for idx, child in enumerate(reversed(node.children)):
if self.get_label(child) != 'ARGUMENTS':
code += self.parse_node(child)
if self.get_label(child) != 'TYPE_REF' and idx < len(node.children) - 1:
code += '.'
code += '('
for child in node.children:
# # for child in node.children:
# # if self.get_label(child) in ['NAME', 'REF', 'REF_BUILTIN']:
if self.get_label(child) == 'ARGUMENTS':
for index, arg in enumerate(child.children):
code += self.parse_node(arg)
if index < len(child.children) - 1:
code += ', '
code += ')'
elif self.get_label(node) == 'TYPE_CALL_EXPR':
for child in node.children:
if self.get_label(child) == 'TYPE_KIND':
code += self.get_type(child.children[0]) + '('
elif self.get_label(child) == 'ARGUMENTS':
for index, arg in enumerate(child.children):
code += self.parse_node(arg)
if index < len(child.children) - 1:
code += ', '
code += ')'
elif self.get_label(node) == 'TYPEDEF_DECL':
for child in node.children:
if self.get_label(child) == 'TYPE_DEF':
code += f'typedef {self.get_type(child.children[0])} '
elif self.get_label(child) == 'IDENTIFIER':
code += self.get_label(child.children[0])
elif self.get_label(node) == 'WHILE_STMT':
code += 'while('
code += self.parse_node(node.children[0])
code += ')'
code += self.parse_node(node.children[1])
elif self.get_label(node) == 'TYPE_REF' or self.get_label(node) == 'TYPE':
code += self.get_type(node)
if self.get_label(node.parent) != 'COMPOUND_LITERAL_EXPR' and not 'CAST' in self.get_label(node.parent):
code += '::'
elif self.get_label(node) == 'CLASS_DECL':
code += 'class '
for child in node.children:
if self.get_label(child) == 'NAME':
code += self.merge_terminals(child.children)
else:
code += self.parse_node(child)
elif self.get_label(node) == 'STRUCT_DECL':
code += 'struct '
for child in node.children:
if self.get_label(child) == 'NAME':
code += self.merge_terminals(child.children)
else:
code += self.parse_node(child)
elif self.get_label(node) in ['BREAK_STMT', 'CONTINUE_STMT']:
code += self.get_label(node).split('_')[0].lower()
elif self.get_label(node) == 'CXX_FOR_RANGE_STMT':
code += 'for('
code += self.parse_node(node.children[0])
code += ':'
code += self.parse_node(node.children[1])
code += ')'
for child in node.children[2:]:
code += self.parse_node(child)
elif self.get_label(node) == 'INIT_LIST_EXPR':
code += '{'
for index, child in enumerate(node.children):
code += self.parse_node(child)
if index < len(node.children) - 1:
code += ','
code += '}'
elif self.get_label(node) == 'DECL_STMT':
for index, child in enumerate(node.children):
code += self.parse_node(child)
if index < len(node.children) - 1:
code += ','
elif self.get_label(node) == 'TEMPLATE_DECL':
code += 'template<'
for index, child in enumerate(node.children):
code += self.parse_node(child)
if index < len(node.children) - 1:
code += ','
code += '>\n'
elif self.get_label(node) == 'TEMPLATE_TYPE_PARAMETER':
code += self.get_temp_type_param(node)
elif self.get_label(node) == 'CXX_FUNCTIONAL_CAST_EXPR':
code += f'{ self.get_type(node.children[0].children[0])}('
for child in node.children[1:]:
code += self.parse_node(child)
code += ')'
elif self.get_label(node) == 'CONSTRUCTOR_INITIALIZER':
constr_inits = []
for child in node.parent.children:
if self.get_label(child) == 'CONSTRUCTOR_INITIALIZER':
constr_inits.append(child)
# If first constr init, add ' : '
if node == constr_inits[0]:
code += ' : '
code += self.parse_node(node.children[0]) + '('
if len(node.children) > 1:
code += self.parse_node(node.children[1])
code += ')'
# If not last constr init, place comma in between constr inits
if node != constr_inits[-1]:
code += ', '
elif self.get_label(node) == 'PACK_EXPANSION_EXPR':
for child in node.children:
code += self.parse_node(child)
code += '...'
elif self.get_label(node) == 'CXX_THIS_EXPR':
if self.get_label(node.parent) == 'MEMBER_REF_EXPR':
code += 'this->'
else:
code += 'this'
elif self.get_label(node) == 'SWITCH_STMT':
code += 'switch('
code += self.parse_node(node.children[0])
code += ')'
for child in node.children[1:]:
code += self.parse_node(child)
elif self.get_label(node) == 'CASE_STMT':
code += 'case '
code += self.parse_node(node.children[0])
code += ':\n'
for child in node.children[1:]:
code += self.parse_node(child)
elif self.get_label(node) == 'CXX_TRY_STMT':
code += 'try'
for child in node.children:
code += self.parse_node(child)
elif self.get_label(node) == 'CXX_CATCH_STMT':
if len(node.children) < 2:
code += 'catch(...)'
for child in node.children:
code += self.parse_node(child)
else:
code += 'catch('
code += self.parse_node(node.children[0])
code += ')'
for child in node.children[1:]:
code += self.parse_node(child)
elif self.get_label(node) == 'LAMBDA_EXPR':
code += self.get_lambda_expr(node)
for child in node.children:
code += self.parse_node(child)
elif self.get_label(node) == 'CXX_STATIC_CAST_EXPR':
code += f'static_cast<{self.get_label(node.children[0].children[0])}>('
for child in node.children[1:]:
code += self.parse_node(child)
code += ')'
elif self.get_label(node) == 'DO_STMT':
code += 'do'
code += self.parse_node(node.children[0])
code += 'while('
for child in node.children[1:]:
code += self.parse_node(child)
code += ')'
elif self.get_label(node) == 'GOTO_STMT':
code += 'goto '
for child in node.children:
code += self.parse_node(child)
elif self.get_label(node) == 'LABEL_STMT':
for label_child in node.children:
if self.get_label(label_child) == 'NAME':
label_name = self.merge_terminals(label_child.children)
code += f'{label_name}:\n'
else:
code += self.parse_node(label_child)
# elif self.get_label(node) == 'TYPE_KIND':
# code += self.get_type(node.children[0])
# print(node.parent, node.children[0].children)
else:
# print(self.get_label(node))
pass
if ('COMPOUND_STMT' in self.get_label(node.parent) \
or self.get_label(node.parent) == 'root' \
# If statement one-liner with no compound statement (= {..}), we still want ";" after each line
or (self.get_label(node.parent) == 'IF_STMT' \
and 'COMPOUND_STMT' not in [self.get_label(c) for c in node.parent.children] \
and node != node.parent.children[0])) \
and (self.get_label(node) != 'FUNCTION_DECL' \
and self.get_label(node) != 'IF_STMT' \
and self.get_label(node) != 'FOR_STMT' \
and self.get_label(node) != 'CXX_FOR_RANGE_STMT' \
and self.get_label(node) != 'WHILE_STMT' \
and self.get_label(node) != 'TEMPLATE_DECL') \
or (self.get_label(node) == 'FUNCTION_DECL' and 'COMPOUND_STMT' not in [self.get_label(c) for c in node.children]):
code += ';\n'
return code
def thread_parser(self, file_queue, pbar, output_folder, imports):
while not file_queue.empty():
ast_id, ast = file_queue.get()
root = self.importer.import_(ast)
output = open(f'{output_folder}{ast_id}.cpp', 'w')
try:
for child in root.children:
output.write(self.parse_node(child))
except Exception as e:
print(f'File: {ast_id} failed: {e}')
pbar.update()
file_queue.task_done()
output.close()
continue
output.close()
# print(imports.keys())
try:
imports_file = imports[int(ast_id)]
imports_file = [ele for ele in imports_file[1:-1].split("'") if ele != '' and ele != ', ']
except KeyError:
imports_file = []
# This is not always added but sometimes needed e.g.: std::cout is used in original code but we simply use cout
if 'using namespace std;' not in imports_file:
imports_file.append('using namespace std;')
add_includes_usings(f'{output_folder}{ast_id}.cpp', imports_file)
pbar.update()
file_queue.task_done()
def parse_asts_to_code(self, input_folder):
file_paths = []
# Read csv file in chunks (may be very large)
asts = pd.read_csv(f'{input_folder}asts0.csv{".bz2" if self.use_compression else ""}', chunksize=1e5)
# Read metadata file with imports
print('loading csv file with imports...')
metadata_file = pd.read_csv(self.csv_file_path, usecols=['solutionId', 'imports'])
metadata_file = metadata_file.set_index('solutionId')
imports = metadata_file.to_dict()['imports']
del metadata_file
# iterate over the chunks
for asts_chunk in asts:
# Create progressbar
pbar = tqdm(total=len(asts_chunk))
# Create file queue to store the program data
file_queue = queue.Queue(len(asts_chunk))
# Fill the queue with files.
for ast in list(asts_chunk.iterrows()):
file_queue.put((ast[1]['id'], ast[1]['AST']))
pbar = tqdm(total=len(file_paths))
try:
threads = []
# List of files with a non-zero return code.
for _ in range(self.processes_num):
t = threading.Thread(target=self.thread_parser,
args=(file_queue, pbar, self.output_folder, imports))
t.daemon = True
t.start()
threads.append(t)
# Wait for all threads to be done.
file_queue.join()
for thread in threads:
thread.join()
except KeyboardInterrupt:
os.kill(0, 9)
"a": "ModuleTwo",
"children": [
{
"a": "ModuleChildrenRightOneOne"
},
{
"a": "ModuleChildrenRightOneTwo"
}
]
}
]
}'''
this_mod = sys.modules[__name__]
root = importer.import_(data)
exporter = DictExporter()
def call_module(node_dict, node):
for value in node_dict.values():
module_obj = getattr(this_mod, value)()
print(module_obj.test(node.parent))
def run_manual(nodes_list):
print('level: {}'.format(nodes_list[0].depth))
for node in nodes_list:
node_dict = exporter.export(node)
if 'children' in node_dict:
def _restore_json(self, string):
'''restore the tree from json'''
imp = JsonImporter()
root = imp.import_(string)
self._debug('Catalog imported from json \"{}\"'.format(self.path))
return root
def tree_from_file(file):
importer = JsonImporter()
data = open(file, 'r').read()
root = importer.import_(data)
return root