예제 #1
0
class EvidenceRpcClient(object):
    def __init__(self, coupled=False):
        self.tree_importer = JsonImporter()
        self.connection = pika.BlockingConnection(
            pika.ConnectionParameters(host='localhost'))
        self.channel = self.connection.channel()
        self.coupled = coupled

        result = self.channel.queue_declare(exclusive=True)
        self.callback_queue = result.method.queue

        self.channel.basic_consume(self.on_response,
                                   no_ack=True,
                                   queue=self.callback_queue)

    def deserialize(self, response):
        res = json.loads(response)
        if self.coupled:
            res['tree'] = self.tree_importer.import_(res['tree'])
        else:
            res[0] = json.loads(res[0])
            res[0]['tree'] = self.tree_importer.import_(res[0]['tree'])

        return res

    def on_response(self, ch, method, props, body):
        if self.corr_id == props.correlation_id:
            self.response = body

    def call(self, request):
        self.response = None
        self.corr_id = str(uuid.uuid4())
        try:
            self.channel.basic_publish(exchange='',
                                       routing_key='rpc_queue',
                                       properties=pika.BasicProperties(
                                           reply_to=self.callback_queue,
                                           correlation_id=self.corr_id,
                                       ),
                                       body=str(request))
        except:
            time.sleep(5)
            self.channel.basic_publish(exchange='',
                                       routing_key='rpc_queue',
                                       properties=pika.BasicProperties(
                                           reply_to=self.callback_queue,
                                           correlation_id=self.corr_id,
                                       ),
                                       body=str(request))

        while self.response is None:
            self.connection.process_data_events()

        return self.deserialize(self.response)
예제 #2
0
def get_rank_kcr(dimacs_, solset_, jsonfile_):
    # read tree structure from file
    _treefile = os.path.dirname(dimacs_) + '/smarch/tree.json'
    # node = AnyNode(count=-1, cube=[])

    if os.path.exists(jsonfile_):
        with open(jsonfile_, 'r') as file:
            data = file.read()
            importer = JsonImporter()
            _root = importer.import_(data)
            total = _root.count
    else:
        print("ERROR: tree file not found!")
        return

    for sol in solset_:
        # traverse tree based on solution
        _node = _root
        _precision = 0
        _number = 0
        while _precision == 0:
            _node, _precision, _number = traverse_cube(_node, sol, _number)

        if _precision > 0:
            print(str(_number / total) + "," + str(_precision / total))
        else:
            print("ERROR: tree traverse failure")
예제 #3
0
    def _loadTree(self, filename, erease):
        str_tree = None
        index = 0

        if filename is None or not os.path.exists(filename):
            return None

        if erease:
            with open(filename, 'w') as file:
                file.write("")

        with open(filename, 'r') as file:
            try:
                index = int(file.readline())
            except ValueError:
                return None
            str_tree = file.read()

        importer = JsonImporter()
        try:
            tree = importer.import_(str_tree)
        except json.JSONDecodeError:
            return None

        return index, tree
예제 #4
0
 def _restore_json(self, string):
     '''restore the tree from json'''
     imp = JsonImporter()
     root = imp.import_(string)
     if self.verbose:
         Logger.info('Catalog imported from json \"{}\"'.format(self.path))
     return root
예제 #5
0
def ast_json_to_anytree(json_data):
    """
    Deserializes an anytree in json format and returns its root node.
    """
    from anytree.importer import JsonImporter
    importer = JsonImporter()
    return importer.import_(json_data)
예제 #6
0
def get_rank(dimacs_, dir_, jsonfile_):
    # read dimacs file for feature list
    _features, _clauses, _vars = read_dimacs(dimacs_)

    if os.path.exists(jsonfile_):
        with open(jsonfile_, 'r') as file:
            data = file.read()
            importer = JsonImporter()
            _root = importer.import_(data)
            total = _root.count
    else:
        print("ERROR: tree file not found!")
        return

    _cdir = dir_
    for file in os.listdir(_cdir):
        if file.endswith('.config'):
            # convert config file into variable list
            sol = read_config_kmax(_features, _cdir + "/" + file)

            # traverse tree based on solution
            _node = _root
            _precision = 0
            _number = 0
            while _precision == 0:
                _node, _precision, _number = traverse_cube(_node, sol, _number)

            if _precision > 0:
                print(str(_number / total) + "," + str(_precision / total))
            else:
                print("ERROR: tree traverse failure")
예제 #7
0
    def __init__(self, vectorifier=None):

        dict_importer = DictImporter(nodecls=Node)
        importer = JsonImporter(dictimporter=dict_importer)
        self.root = importer.import_(symptom_json)

        if vectorifier is not None:
            # normal mode, if it is None is eval mode
            self.vectorifier = vectorifier
            self.vector_dimension = vectorifier.d
            """
      if concept name vectors file not exists, create it
      """
            if vectorifier.internal_representation == "glove":
                if not os.path.isfile(COM.CSV_ST_CONCEPT_NAMES_GLOVE_PATH + \
                                      str(self.vector_dimension) + "d.csv"):
                    print(
                        "Concept names glove vectors file not found. \nComputing file..."
                    )
                    self.__save_csv_concept_name_glove_vectors()

                self.concept_name_vectors_df = pd.read_csv(COM.CSV_ST_CONCEPT_NAMES_GLOVE_PATH + \
                                                           str(self.vector_dimension) + "d.csv",
                                                           header=None)
            elif vectorifier.internal_representation == "bert":
                if not os.path.isfile(COM.CSV_ST_CONCEPT_NAMES_BERT_PATH + \
                                      COM.FILENAME_CSV_ST_CONCEPT_NAMES_BERT):
                    print(
                        "Concept names bert vectors file not found. \nComputing file..."
                    )
                    self.__save_csv_concept_name_bert_vectors()

                self.concept_name_vectors_df = pd.read_csv(COM.CSV_ST_CONCEPT_NAMES_BERT_PATH + \
                                                           COM.FILENAME_CSV_ST_CONCEPT_NAMES_BERT,
                                                           header=None)
예제 #8
0
 def get_tree(self, file_name):
     ''' This function is used to import json file and return anytree Node '''
     importer = JsonImporter()
     tree_file = open(file_name, "r")
     data = tree_file.read()
     tree_file.close()
     return importer.import_(data)
예제 #9
0
def read_tree(file_path: str):
    """Deserialize a tree from a saved JSON file"""
    importer = JsonImporter()
    with open(file_path, 'r') as j:
        json_tree = j.read().replace("_name", "name")
    root = importer.import_(json_tree)
    root.cost = None
    return root
예제 #10
0
def _deserialize_ontologies(filename):
  """Deserializes an ontology from a JSON file and returns its root."""
  importer = JsonImporter()
  forest = []
  with open(filename, 'r') as f:
    tree_list = json.load(f)
    for tree_text in tree_list:
      forest.append(importer.import_(tree_text))
  return forest
예제 #11
0
 def read(self) -> Entry:
     content = None
     if os.path.isfile(self._filepath):
         fp = open(Path(self._filepath))
         d_imp = DictImporter(nodecls=Entry)
         importer = JsonImporter(dictimporter=d_imp)
         content = importer.import_(fp.read())
         fp.close()
     return content
예제 #12
0
def draw_tree(string1, path_des, file):
    from anytree import RenderTree
    importer = JsonImporter()
    root = importer.import_(string1)
    # Render Tree
    print(RenderTree(root, style=ContRoundStyle()))
    # # Render graph tree
    DotExporter(root).to_picture(path_des + file)
    Image(filename=path_des + file)
예제 #13
0
 def load(self):
     extension = ".rpt"
     dir_name = "../data/policies/"
     file_name = dir_name + self.agent_name + extension
     importer = JsonImporter()
     data = None
     with open(file_name, "r") as f:
         data = json.loads(f)
         f.close()
     self.tree = importer.import_(data)
    def load_tree(self):
        with open('test.json') as json_file:
            data = json.load(json_file)

        print(data)
        print("\n")
        data_json = json.dumps(data)
        importer = JsonImporter()
        d0 = importer.import_(data_json)
        print(RenderTree(d0))
예제 #15
0
def import_conversation_trees_from_db(user_id):
    documents = collection_trees.find()
    trees = []
    for i, tree in enumerate(documents):
        importer = JsonImporter()
        r1 = json_util.dumps(tree)
        root = importer.import_(r1)
        if i % 1000 == 0:
            print("Finding: {}".format(i))
        trees.append(root)
    return trees
예제 #16
0
    def __init__(self, dir='controllers/'):

        importer = JsonImporter()

        with open(dir + 'data.json', 'r') as f:
            data = json.load(f)
            self.root = importer.import_(data)

        self.current_node = self.root

        DotExporter(self.root).to_picture("tree.png")
예제 #17
0
def printTree(tweetsFile):
    propTree = PropTree()  # an instance of a tree
    importer = JsonImporter()
    rootNr = 0
    with open(tweetsFile, 'r') as _file:
        content = _file.read()

    contentSplit = content.split("&")
    contentSplit.pop()
    for data in contentSplit:
        root = importer.import_(data)
        propTree.addRoot(root)
        rootNr += 1
    return propTree
예제 #18
0
def drawtree(string, path_des, path, filename):
    try:
        error_flag = 0
        importer = JsonImporter()
        root = importer.import_(string)
        print(RenderTree(root, style=ContRoundStyle()))
        DotExporter(root).to_picture(path_des + filename)
        Image(filename=path_des + filename)
        return (error_flag)
    except:
        error_flag = 1
        f = open(path + '/H_log.dat', 'a+')
        f.write(filename + '\tInvalid JSON format\n')
        f.close()
        return (error_flag)
def printTree(tweetsFile):
    propTree = PropTree()  # an instance of a tree
    importer = JsonImporter()
    rootNr = 0

    with open('./data/tree/trees/' + tweetsFile + '.txt', 'r') as _file:
        content = _file.read()

    contentSplit = content.split("&")

    for data in contentSplit:
        root = importer.import_(data)
        propTree.addRoot(root)
        rootNr += 1

    propTree.makeSimpleTree()
    return propTree
예제 #20
0
def drawtree(string, path_des, path, filename, file):
	try:
		error_flag = 0
		importer = JsonImporter()
		root = importer.import_(string)
		file1 = file+'.dot'
		print(RenderTree(root, style=ContRoundStyle()))
		DotExporter(root).to_dotfile(path_des+file1)
		add_edge_labels(path_des, file1)
		check_call(['dot','-Tpng',path_des+file1,'-o',path_des+file+'.png'])
		return(error_flag)
	except:
		error_flag = 1
		f = open(path+'/E_log.dat', 'a+')
		f.write(filename +'\tInvalid Drawtree input format\n')
		f.close()
		f = open(path_des+'/E_log.dat', 'a+')
		f.write('\tInvalid Drawtree input format\n')
		f.close()
		return(error_flag)
예제 #21
0
def drawtree(string, path_des, filename):
	importer = JsonImporter()
	root = importer.import_(string)
	print(RenderTree(root, style=ContRoundStyle()))
	DotExporter(root).to_picture(path_des+filename)
	Image(filename=path_des+filename)
예제 #22
0
def JSON_toTree(json_data):
    importer = JsonImporter()
    ITSM_tree = importer.import_(json_data)
    print(RenderTree(ITSM_tree))
    return ITSM_tree
예제 #23
0
def get_rank_unigen(dimacs_, sampleFile_):
    # read dimacs file for feature list
    _features, _clauses, _vars = read_dimacs(dimacs_)

    # read tree structure from file
    _treefile = os.path.dirname(dimacs_) + '/smarch/tree.json'
    # node = AnyNode(count=-1, cube=[])

    if os.path.exists(_treefile):
        with open(_treefile, 'r') as file:
            data = file.read()
            importer = JsonImporter()
            _root = importer.import_(data)
            total = _root.count
    else:
        print("ERROR: tree file not found!")
        return

    with open(sampleFile_, 'r') as f:
        i = 1
        for line in f:
            line = line[1:len(line)]
            raw = line.split()
            if len(raw) != 0:
                sol = raw[1:len(raw) - 1]
                #print(sol)
                # traverse tree based on solution
                _node = _root
                _precision = 0
                _number = 0

                _cdir = os.path.dirname(dimacs_) + '/'
                sdimacs = _cdir + "/sample.dimacs"
                ssol = _cdir + "/" + "sample.sol"
                gen_dimacs(_vars, _clauses, sol, sdimacs)
                getoutput("minisat " + sdimacs + " " + ssol)

                _outfile = os.path.dirname(sampleFile_) + '/validCheck.dimacs'
                gen_dimacs(_vars, _clauses, sol, _outfile)
                res = getoutput(SHARPSAT + ' -q ' + _outfile)

                fsol = list()
                with open(ssol, 'r') as f:
                    fsol = f.read().split()
                    del fsol[0]

                if is_int(res):
                    print(res, end=',')
                    while _precision == 0:
                        _node, _precision, _number = traverse_cube(
                            _node, fsol, _number)

                    if _precision > 0:
                        print(
                            str(_number / total) + "," +
                            str(_precision / total))
                    else:
                        print("ERROR: tree traverse failure")
                else:
                    print("ERROR: sample invalid")
            i += 1
예제 #24
0
            for pre, fill, node in RenderTree(root):
                fileTreeLog.write("%s%s\n" % (pre, node.name))

            fileTreeLog.close()

        if mode == 'pred' or mode == 'kValid':

            sequences = sequencesVal
            ys = classesVal

            #import tree from json file
            if mode == 'pred':
                with open(treeSource) as infile:
                    jsonTree = json.load(infile)
                importer = JsonImporter()
                root = importer.import_(jsonTree)
            #przeprowadz predykcje na zbiorze
            y_preds = predictBatch(sequences, root)
            y_preds = [
                y_pred[0] if len(y_pred) == 1 else int(np.median(y_pred))
                for y_pred in y_preds
            ]
            answers = {'good': 0, 'bad': 0}
            for y, y_pred in zip(ys, y_preds):
                if y == y_pred:
                    answers['good'] += 1
                else:
                    answers['bad'] += 1
            print("Wynik predykcji: " +
                  str(answers['good'] / (answers['good'] + answers['bad'])))
예제 #25
0
class AstToCodeParser:
    def __init__(self, output_folder, csv_file_path='', use_compression=False, processes_num=1, tokenized=True):
         # CSV to get program data from
        self.csv_file_path = csv_file_path

        # Output folder to save data to
        self.output_folder = output_folder

        self.tokenized = tokenized

        # Create JSON importer
        self.importer = JsonImporter()

        # Boolean indicating whether ASTs from the input folder are stored using compression
        self.use_compression = use_compression

        # Number of parallel processes
        self.processes_num = processes_num

    
    def load_vocabs_from_folder(self, folder):
        if self.tokenized:
            self.tokenizers = {}

            for dirs, _, files in os.walk(folder):
                for file in files:
                    if 'tokens.json' in file:
                        self.tokenizers[file.split('_')[0]] = Tokenizer(self.output_folder, self.tokenized)
                        self.tokenizers[file.split('_')[0]].load_from_path(os.path.join(dirs, file))


    def load_vocabs_from_dicts(self, token2index_dicts):
        if self.tokenized:
            self.tokenizers = {}

            for vocab_type, token2index in token2index_dicts.items():
                self.tokenizers[vocab_type] = Tokenizer(self.output_folder, self.tokenized)
                self.tokenizers[vocab_type].load_from_dict(token2index)



    def get_label(self, node):
        if self.tokenized:
            if node.res:
                return self.tokenizers['RES'].get_label(node.token)
            else:
                parent_label = self.tokenizers['RES'].get_label(node.parent.token)
                if 'LITERAL' in parent_label: 
                    return self.tokenizers['LITERAL'].get_label(node.token)      
                elif 'TYPE' == parent_label:
                    return self.tokenizers['TYPE'].get_label(node.token)
                elif 'REF_BUILTIN' == parent_label:
                    return self.tokenizers['NAME_BUILTIN'].get_label(node.token)
                else:
                    return self.tokenizers['NAME'].get_label(node.token)
        else:
            return node.token

    def merge_terminals(self, terminals):
        return ' '.join([self.get_label(term) for term in terminals])

    def get_operator(self, ast_item):
        operator = self.get_label(ast_item).split('_')[-1]
        code = ''
        if 'BINARY' in self.get_label(ast_item) or 'COMPOUND_ASSIGNMENT' in self.get_label(ast_item):
            if len(ast_item.children) > 0:
                code += self.parse_node(ast_item.children[0])

            code += f' {operator} '

            if len(ast_item.children) > 1:
                code += self.parse_node(ast_item.children[1])

        elif 'UNARY' in self.get_label(ast_item):
            if 'POST' in self.get_label(ast_item):
                if len(ast_item.children) > 0:
                    code += self.parse_node(ast_item.children[0])
                code += operator
            else:
                code += operator + ' '

                if len(ast_item.children) > 0:
                    code += self.parse_node(ast_item.children[0])
        elif 'CONDITIONAL_OPERATOR' == self.get_label(ast_item):
            if len(ast_item.children) > 0:
                code += self.parse_node(ast_item.children[0])
            code += ' ? '

            if len(ast_item.children) > 1:
                code += self.parse_node(ast_item.children[1])

            code += ':'

            if len(ast_item.children) > 2:
                code += self.parse_node(ast_item.children[2])
        else:
            print(ast_item)
            pass



        return code


    def get_call_exp_operator(self, ast_item):
        code = ''
        operator_label = self.get_label(ast_item.children[0].children[0])
        operator = operator_label.split('operator')[-1]

        if operator == '[]':
            if len(ast_item.children[1].children) > 0:
                code += self.parse_node(ast_item.children[1].children[0])

            if len(ast_item.children[1].children) > 1:
                code += f'[{self.parse_node(ast_item.children[1].children[1])}]'
        elif operator == '()':

            if len(ast_item.children[1].children) > 0:
                code += self.parse_node(ast_item.children[1].children[0])
            if len(ast_item.children[1].children) > 1:
                code += f'({self.parse_node(ast_item.children[1].children[1])})'
            else:
                code += '()'
        elif operator == '*' or (len(ast_item.children) > 1 and len(ast_item.children[1].children) < 2):
            code += operator

            if len(ast_item.children[1].children) > 0:
                code += self.parse_node(ast_item.children[1].children[0])
        elif operator in ['++_PRE', '++_POST', '--_PRE', '--_POST']:
            if 'PRE' in operator:
                code += operator.split('_')[0]
                if len(ast_item.children[1].children) > 0:
                    code += self.parse_node(ast_item.children[1].children[0])
            else:
                if len(ast_item.children[1].children) > 0:
                    code += self.parse_node(ast_item.children[1].children[0])
                code += operator.split('_')[0]
        else:
            # first argument of operator
            if len(ast_item.children) > 1 and len(ast_item.children[1].children) > 0:
                code += self.parse_node(ast_item.children[1].children[0])
            code += f' {operator} '

            if len(ast_item.children) > 1 and len(ast_item.children[1].children) > 1:
                code += self.parse_node(ast_item.children[1].children[1])        

        return code


    def get_var_decl(self, ast_item):
        # Contains all non-ref and non-type values
        # e.g. int ch[1][1] = {{2}} would give [1, 1, 2]
        declarations = []
        acc_spec = ''
        var_type = ''
        var_name = 'VAR_NAME_PLACEHOLDER'

        # If the variable is an array, this is will be filled
        array_sizes = []


        for child in ast_item.children:
            # Skip the type if this is a child of a declaration statement and not the first child
            #  and the type is the same as the type of the first child
            if self.get_label(child) == 'TYPE_KIND' and \
            not (self.get_label(ast_item.parent) == 'DECL_STMT' \
                and ast_item != ast_item.parent.children[0]) and len(child.children) > 0:
                var_type_size = self.get_type(child.children[0])

                if type(var_type_size) is tuple:
                    var_type += var_type_size[0]
                    array_sizes = var_type_size[1]
                else:
                    var_type = var_type_size
            elif self.get_label(child) == 'TYPE_KIND':
                var_type_size = self.get_type(child.children[0])

                if type(var_type_size) is tuple:
                    array_sizes = var_type_size[1]

              
            elif self.get_label(child) == 'DECLARATOR':
                for decl_child in child.children:
                    if self.get_label(decl_child) == 'NAME':
                        var_name = self.merge_terminals(decl_child.children)
                    else:
                        declarations.append(self.parse_node(decl_child))
            elif self.get_label(child) == 'ACCESS_SPECIFIER':
                acc_spec += f'{self.merge_terminals(child.children).lower()}:\n'        

        ref_dims = ['[' + num + ']' for num in array_sizes]

        # initialize value the variable was initialized to (empty)
        var_value = ''

        # append initial values if set
        if len(declarations) > 0:
            var_value += ' = ' + ' '.join(declarations)

        # Combine the type of the variable with the name, dimensions and initial value
        return f'{acc_spec}{var_type} {var_name}{"".join(ref_dims)}{var_value}'

    def get_type(self, ast_item):
        type_string = ''

        if self.get_label(ast_item) == 'POINTER':
            type_string += self.get_type(ast_item.children[0])

            type_string += ' *'
            return type_string

        elif self.get_label(ast_item) == 'CONST_QUALIFIED':
            type_string += 'const '
            type_string += self.get_type(ast_item.children[0])
            return type_string


        elif self.get_label(ast_item) in ['TYPE', 'TYPE_REF']:
            if len(ast_item.children) > 0:
                return self.get_label(ast_item.children[0])
            else:
                return ''

        elif self.get_label(ast_item) == 'TYPE_ARRAY':
            array_sizes = []
            array_type = 'auto'
            for array_child in ast_item.children:
                if self.get_label(array_child) in ['TYPE', 'TYPE_REF', 'TYPE_RECORD']:
                    array_type = self.get_type(array_child)
                elif self.get_label(array_child) == 'ARRAY_SIZES':
                    for array_size in array_child.children:
                        array_sizes.append(self.parse_node(array_size))

            type_string += array_type

            return (type_string, array_sizes)

        elif self.get_label(ast_item) == 'TYPE_RECORD':
            record_type = 'auto'
            for record_child in ast_item.children:
                if self.get_label(record_child) in ['TYPE', 'TYPE_REF']:
                    record_type = self.get_type(record_child)   

            type_string += record_type

            for record_child in ast_item.children:                       
                if self.get_label(record_child) == 'TYPE_RECORD_ELEMENTS':
                    type_string += '<'
                    for index, record_element in enumerate(record_child.children):
                        type_string += self.get_type(record_element)

                        if index < len(record_child.children)- 1:
                            type_string += ', '

                    type_string += '>'

            
            return type_string

        elif self.get_label(ast_item) == 'LVALUEREFERENCE':
            return self.get_type(ast_item.children[0]) + '&'

        elif self.get_label(ast_item) == 'RVALUEREFERENCE':
            return self.get_type(ast_item.children[0]) + '&&'

        else:
            return self.get_label(ast_item)

                            
                    


    def get_function_decl(self, ast_item):
        params = []
        acc_spec = ''
        return_type = ''
        const = ''
        func_name = ''
        for child in ast_item.children:
            if self.get_label(child) == 'TYPE_KIND':
                return_type += self.get_type(child.children[0])
            elif self.get_label(child) == 'PARM_DECL':
                params.append(self.get_var_decl(child))
            elif self.get_label(child) == 'NAME':
                func_name = self.merge_terminals(child.children)
            elif self.get_label(child) == 'ACCESS_SPECIFIER':
                acc_spec += f'{self.get_label(child.children[0]).lower()}:\n'
            elif self.get_label(child) == 'CONST':
                const += ' const'

        
        return f'{acc_spec}{return_type} {func_name}({", ".join(params)}){const}'

    def is_function(self, node):
        return self.get_label(node) in ['FUNCTION_DECL', 'FUNCTION_TEMPLATE',
                            'CXX_METHOD', 'DESTRUCTOR', 'CONSTRUCTOR']


    def get_temp_type_param(self, ast_item):
        parameter_pack = False
        # If it is the last template type parameter, check if we need to add ... (=parameter pack -> zero or more params)
        if ast_item == ast_item.parent.children[-1]:
            function_decl_index = -1
            for index, child in enumerate(ast_item.parent.parent.children):
                if child == ast_item.parent:
                    function_decl_index = index + 1

            if len(ast_item.parent.children) > function_decl_index and len(ast_item.parent.children[function_decl_index].children) > 1:
                last_parm_decl = ast_item.parent.parent.children[function_decl_index].children[-2].children[0].children[0]
                if self.get_label(last_parm_decl).endswith('...'):
                    parameter_pack = True

        out = f'typename{"..." if parameter_pack else ""}'

        if len(ast_item.children) > 0:
            return out + f'{self.get_label(ast_item.children[0])}'
        else:
            return out


    def get_lambda_expr(self, ast_item):
        capture_clauses = []
        params = []

        for child in ast_item.children:
            if self.get_label(child) == 'CAPTURE_CLAUSE':
                capture_clauses.append(self.merge_terminals(child.children))
            elif self.get_label(child) == 'PARM_DECL':
                params.append(self.get_var_decl(child))

        
        return f'[{", ".join(capture_clauses)}]({", ".join(params)})'


    def parse_node(self, node):
        call_exp_operators = ['[]', '=', '<<', '>>', '==', '+', '-', '%', '*', '/',
                             '+=', '-=', '^=', '||', '()', '!=',
                             '++_PRE', '++_POST', '--_PRE', '--_POST']

        call_exp_operator_labels = ['operator' + op for op in call_exp_operators]
        code = ''

        if self.get_label(node) == 'VAR_DECL' or self.get_label(node) == 'FIELD_DECL' or self.get_label(node) == 'UNEXPOSED_DECL':
            code += self.get_var_decl(node)
        elif self.is_function(node):
            code += self.get_function_decl(node)
            for child in node.children:
                code += self.parse_node(child)
        elif self.get_label(node) == 'COMPOUND_STMT':
            code += ' {\n'
            for child in node.children:
                code += self.parse_node(child)
            code += '}\n'
        elif self.get_label(node) == 'PAREN_EXPR':
            code += '('
            for child in node.children:
                code += self.parse_node(child)
            code += ')'
        elif 'OPERATOR' in self.get_label(node) or 'UNARY_EXPR' in self.get_label(node):
            code += self.get_operator(node)
        elif self.get_label(node) == 'GNU_NULL_EXPR':
            code += 'NULL'
        elif self.get_label(node) == 'CXX_NULL_PTR_LITERAL_EXPR':
            code += 'nullptr'
        elif self.get_label(node) == 'COMPOUND_LITERAL_EXPR' or self.get_label(node) == 'CSTYLE_CAST_EXPR':
            code += '('
            code += self.parse_node(node.children[0]) if self.get_label(node) == 'COMPOUND_LITERAL_EXPR' else self.get_type(node.children[0].children[0])
            code += ')'
            for child in node.children[1:]:
                code += self.parse_node(child)
        elif self.get_label(node) in ['DECL_REF_EXPR', 'MEMBER_REF_EXPR', 'MEMBER_REF', 'LABEL_REF', 'REF', 'REF_BUILTIN'] or 'LITERAL' in self.get_label(node):
            for child in node.children[1:]:
                code += self.parse_node(child)
                code += '.'

            if self.get_label(node) == 'MEMBER_REF_EXPR' and len(node.children) > 1 \
                and self.get_label(node.children[1]) != 'CXX_THIS_EXPR': #node.parent.parent) == 'REF':
                code += '.'

            if len(node.children) > 0:
                code += self.get_label(node.children[0])

            # if self.get_label(node.parent) in ['REF', 'REF_BUILTIN']:
            #     code += '.'


        elif self.get_label(node) == 'IF_STMT':
            code += 'if('
            # The first compound statement of an if statement is the code of the
            # if statement, the second (and also last) compound statement is for
            # a possible else statement
            compound_statements = 0
            for index, child in enumerate(node.children):
                if index == len(node.children) - 1 and compound_statements > 0:
                    code += 'else '
                if self.get_label(child) == 'COMPOUND_STMT':
                    compound_statements += 1

                code += self.parse_node(child)

                if index == 0:
                    code += ')'

        elif self.get_label(node) == 'ARRAY_SUBSCRIPT_EXPR':
            if len(node.children) > 0:
                code += self.parse_node(node.children[0])
            code += '['

            if len(node.children) > 1:
                code += self.parse_node(node.children[1])
                
            code += ']'
        elif self.get_label(node) == 'RETURN_STMT':
            code += 'return'
            for child in node.children:
                code += ' '
                code += self.parse_node(child)
        elif self.get_label(node) == 'FOR_STMT':
            code += 'for('

            # Count amount of expressions in for loop (can essentialy be empty: for(;;))
            for_stmt_expressions = 0

            for child in node.children:
                if self.get_label(child) != 'COMPOUND_STMT':
                    for_stmt_expressions += 1
                else:
                    break

            # add ; for empty expressions e.g. if we have 2 expressions: for(;expr2;expr3)
            for _ in range(3 - for_stmt_expressions):
                code += ';'

            # add expressions
            for i in range(for_stmt_expressions):
                code += self.parse_node(node.children[i])
                if i < for_stmt_expressions - 1:
                    code += '; '
            code += ')'

            # parse for loop compound statement
            for i in range(for_stmt_expressions, len(node.children)):
                code += self.parse_node(node.children[i])
        elif self.get_label(node) == 'CALL_EXPR':
            if len(node.children) > 0 and len(node.children[0].children) > 0 and self.get_label(node.children[0].children[0]) in call_exp_operator_labels:
                code += self.get_call_exp_operator(node)

            else:
                for idx, child in enumerate(reversed(node.children)):
                    if self.get_label(child) != 'ARGUMENTS':
                        code += self.parse_node(child)
                        if self.get_label(child) != 'TYPE_REF' and idx < len(node.children) - 1:
                            code += '.'

                code += '('

                for child in node.children:                        
                # # for child in node.children:
                # #     if self.get_label(child) in ['NAME', 'REF', 'REF_BUILTIN']:
                    if self.get_label(child) == 'ARGUMENTS':
                        for index, arg in enumerate(child.children):
                            code += self.parse_node(arg)
                            if index < len(child.children) - 1:
                                code += ', '
                code += ')'

        elif self.get_label(node) == 'TYPE_CALL_EXPR':
            for child in node.children:
                if self.get_label(child) == 'TYPE_KIND':
                    code += self.get_type(child.children[0]) + '('
                elif self.get_label(child) == 'ARGUMENTS':
                    for index, arg in enumerate(child.children):
                        code += self.parse_node(arg)
                        if index < len(child.children) - 1:
                            code += ', '
            code += ')'
        
        elif self.get_label(node) == 'TYPEDEF_DECL':
            for child in node.children:
                if self.get_label(child) == 'TYPE_DEF':
                    code += f'typedef {self.get_type(child.children[0])} '
                elif self.get_label(child) == 'IDENTIFIER':
                    code += self.get_label(child.children[0])

        elif self.get_label(node) == 'WHILE_STMT':
            code += 'while('
            code += self.parse_node(node.children[0])
            code += ')'
            code += self.parse_node(node.children[1])

        elif self.get_label(node) == 'TYPE_REF' or self.get_label(node) == 'TYPE':
            code += self.get_type(node)
            if self.get_label(node.parent) != 'COMPOUND_LITERAL_EXPR' and not 'CAST' in self.get_label(node.parent):
                code += '::'

        elif self.get_label(node) == 'CLASS_DECL':
            code += 'class '
            for child in node.children:
                if self.get_label(child) == 'NAME':
                    code += self.merge_terminals(child.children)
                else:
                    code += self.parse_node(child)

        elif self.get_label(node) == 'STRUCT_DECL':
            code += 'struct '
            for child in node.children:
                if self.get_label(child) == 'NAME':
                    code += self.merge_terminals(child.children)
                else:
                    code += self.parse_node(child)

        elif self.get_label(node) in ['BREAK_STMT', 'CONTINUE_STMT']:
            code += self.get_label(node).split('_')[0].lower()
        elif self.get_label(node) == 'CXX_FOR_RANGE_STMT':
            code += 'for('
            code += self.parse_node(node.children[0])
            code += ':'
            code += self.parse_node(node.children[1])
            code += ')'

            for child in node.children[2:]:
                code += self.parse_node(child)

        elif self.get_label(node) == 'INIT_LIST_EXPR':
            code += '{'
            for index, child in enumerate(node.children):
                code += self.parse_node(child)
                if index < len(node.children) - 1:
                    code += ','
            code += '}'
        elif self.get_label(node) == 'DECL_STMT':
            for index, child in enumerate(node.children):
                code += self.parse_node(child)
                if index < len(node.children) - 1:
                    code += ','
        elif self.get_label(node) == 'TEMPLATE_DECL':
            code += 'template<'
            for index, child in enumerate(node.children):
                code += self.parse_node(child)
                if index < len(node.children) - 1:
                    code += ','
            code += '>\n'

        elif self.get_label(node) == 'TEMPLATE_TYPE_PARAMETER':
            code += self.get_temp_type_param(node)
        elif self.get_label(node) == 'CXX_FUNCTIONAL_CAST_EXPR':
            code += f'{ self.get_type(node.children[0].children[0])}('
            for child in node.children[1:]:
                code += self.parse_node(child)
            
            code += ')'

        elif self.get_label(node) == 'CONSTRUCTOR_INITIALIZER':
            constr_inits = []

            for child in node.parent.children:
                if self.get_label(child) == 'CONSTRUCTOR_INITIALIZER':
                    constr_inits.append(child)

            # If first constr init, add ' : '
            if node == constr_inits[0]:
                code += ' : '
            
            code += self.parse_node(node.children[0]) + '('
            if len(node.children) > 1:
                code += self.parse_node(node.children[1])
            code += ')'

            # If not last constr init, place comma in between constr inits
            if node != constr_inits[-1]:
                code += ', '

        elif self.get_label(node) == 'PACK_EXPANSION_EXPR':
            for child in node.children:
                code += self.parse_node(child)
            code += '...'

        elif self.get_label(node) == 'CXX_THIS_EXPR':
            if self.get_label(node.parent) == 'MEMBER_REF_EXPR':
                code += 'this->'
            else:
                code += 'this'


        elif self.get_label(node) == 'SWITCH_STMT':
            code += 'switch('
            code += self.parse_node(node.children[0])
            code += ')'
            for child in node.children[1:]:
                code += self.parse_node(child)

        elif self.get_label(node) == 'CASE_STMT':
            code += 'case '
            code += self.parse_node(node.children[0])
            code += ':\n'
            for child in node.children[1:]:
                code += self.parse_node(child)


        elif self.get_label(node) == 'CXX_TRY_STMT':
            code += 'try'
            for child in node.children:
                code += self.parse_node(child)

        elif self.get_label(node) == 'CXX_CATCH_STMT':
            if len(node.children) < 2:
                code += 'catch(...)'
                for child in node.children:
                    code += self.parse_node(child)
            
            else:
                code += 'catch('
                code += self.parse_node(node.children[0])
                code += ')'
                for child in node.children[1:]:
                    code += self.parse_node(child)

        elif self.get_label(node) == 'LAMBDA_EXPR':
            code += self.get_lambda_expr(node)
            for child in node.children:
                code += self.parse_node(child)

        elif self.get_label(node) == 'CXX_STATIC_CAST_EXPR':
            code += f'static_cast<{self.get_label(node.children[0].children[0])}>('
            for child in node.children[1:]:
                code += self.parse_node(child)

            code += ')'


        elif self.get_label(node) == 'DO_STMT':
            code += 'do'
            code += self.parse_node(node.children[0])
            code += 'while('
            for child in node.children[1:]:
                code += self.parse_node(child)
            code += ')'

        elif self.get_label(node) == 'GOTO_STMT':
            code += 'goto '
            for child in node.children:
                code += self.parse_node(child)

        elif self.get_label(node) == 'LABEL_STMT':
            for label_child in node.children:
                if self.get_label(label_child) == 'NAME':
                    label_name = self.merge_terminals(label_child.children)
                    code += f'{label_name}:\n'
                else:
                    code += self.parse_node(label_child)

        # elif self.get_label(node) == 'TYPE_KIND':
        #     code += self.get_type(node.children[0])
        #     print(node.parent, node.children[0].children)

        else:
            # print(self.get_label(node))
            pass

        if ('COMPOUND_STMT' in self.get_label(node.parent) \
        or self.get_label(node.parent) == 'root' \
        # If statement one-liner with no compound statement (= {..}), we still want ";" after each line
        or (self.get_label(node.parent) == 'IF_STMT' \
        and 'COMPOUND_STMT' not in [self.get_label(c) for c in node.parent.children] \
        and node != node.parent.children[0])) \
        and (self.get_label(node) != 'FUNCTION_DECL' \
        and self.get_label(node) != 'IF_STMT' \
        and self.get_label(node) != 'FOR_STMT' \
        and self.get_label(node) != 'CXX_FOR_RANGE_STMT' \
        and self.get_label(node) != 'WHILE_STMT' \
        and self.get_label(node) != 'TEMPLATE_DECL') \
        or (self.get_label(node) == 'FUNCTION_DECL' and 'COMPOUND_STMT' not in [self.get_label(c) for c in node.children]):
            code += ';\n'

        return code


    def thread_parser(self, file_queue, pbar, output_folder, imports):
        while not file_queue.empty():
            ast_id, ast = file_queue.get()

            root = self.importer.import_(ast)

            output =  open(f'{output_folder}{ast_id}.cpp', 'w')
                
            try:
                for child in root.children:
                    output.write(self.parse_node(child))
            except Exception as e:
                print(f'File: {ast_id} failed: {e}')
                pbar.update()
                file_queue.task_done()
                output.close()
                continue

            output.close()

            # print(imports.keys())
            try:
                imports_file = imports[int(ast_id)]
                imports_file = [ele for ele in imports_file[1:-1].split("'") if ele != '' and ele != ', ']
            except KeyError:
                imports_file = []

            # This is not always added but sometimes needed e.g.: std::cout is used in original code but we simply use cout
            if 'using namespace std;' not in imports_file:
                imports_file.append('using namespace std;')

            add_includes_usings(f'{output_folder}{ast_id}.cpp', imports_file)
            pbar.update()
            file_queue.task_done()


    def parse_asts_to_code(self, input_folder):
        file_paths = []

        # Read csv file in chunks (may be very large)
        asts = pd.read_csv(f'{input_folder}asts0.csv{".bz2" if self.use_compression else ""}', chunksize=1e5)

        # Read metadata file with imports
        print('loading csv file with imports...')
        metadata_file = pd.read_csv(self.csv_file_path, usecols=['solutionId', 'imports'])
        metadata_file = metadata_file.set_index('solutionId')
        imports = metadata_file.to_dict()['imports']
        del metadata_file

        # iterate over the chunks
        for asts_chunk in asts:
            # Create progressbar
            pbar = tqdm(total=len(asts_chunk))
            # Create file queue to store the program data
            file_queue = queue.Queue(len(asts_chunk))

            # Fill the queue with files.
            for ast in list(asts_chunk.iterrows()):
                    file_queue.put((ast[1]['id'], ast[1]['AST']))
        
            pbar = tqdm(total=len(file_paths))

            try:
                threads = []
                # List of files with a non-zero return code.
                for _ in range(self.processes_num):
                    t = threading.Thread(target=self.thread_parser,
                                        args=(file_queue, pbar, self.output_folder, imports))
                    t.daemon = True
                    t.start()
                    threads.append(t)

                # Wait for all threads to be done.
                file_queue.join()
                for thread in threads:
                    thread.join() 
 
            except KeyboardInterrupt:
                os.kill(0, 9)
예제 #26
0
       "a": "ModuleTwo",
       "children": [
         {
           "a": "ModuleChildrenRightOneOne"
         },
         {
            "a": "ModuleChildrenRightOneTwo"
         }
       ]
     }
   ]
 }'''

this_mod = sys.modules[__name__]

root = importer.import_(data)

exporter = DictExporter()


def call_module(node_dict, node):
    for value in node_dict.values():
        module_obj = getattr(this_mod, value)()
        print(module_obj.test(node.parent))


def run_manual(nodes_list):
    print('level: {}'.format(nodes_list[0].depth))
    for node in nodes_list:
        node_dict = exporter.export(node)
        if 'children' in node_dict:
예제 #27
0
 def _restore_json(self, string):
     '''restore the tree from json'''
     imp = JsonImporter()
     root = imp.import_(string)
     self._debug('Catalog imported from json \"{}\"'.format(self.path))
     return root
예제 #28
0
def tree_from_file(file):
    importer = JsonImporter()
    data = open(file, 'r').read()
    root = importer.import_(data)

    return root