Beispiel #1
0
def main(args):
    results_path  = "../ccdetection/AST_parser/parse_results"
    parser_script = "../ccdetection/AST_parser/parser"
    phpjoern_path = "/opt/phpjoern"

    # Setup command line arguments.
    parser = ModeArgsParser()
    setupArgs(parser)
    
    try:
        flow = parser.parseArgs(args[1], args[2:])
        
    except:
        parser.printHelp(args[0])
        sys.exit()

    if flow[parser.KEY_MODE] == ARGS_CONVERT:
        # Convert mode.
        # Take a PHP-file path as input and convert it into a gremlin query.
        parser = ASTParser(results_path, parser_script, phpjoern_path)
        ast_root = parser.runPHPParser(getArg(flow, "p", "path"))

        if not ast_root:
            print "Could not parse given path! Exit."
            sys.exit()
        
        converter = AST2Gremlin(ast_root)
        
        try:
            # getArg triggers an exception, if the parameter -d/--debug
            # was not specified.
            getArg(flow, "d", "debug")
            query = converter.convertPHPAST(debug=True)
            
        except:
            query = converter.convertPHPAST()
            
        print query
Beispiel #2
0
def main(argv):
    # Setup command line arguments.
    parser = ModeArgsParser()
    setupArgs(parser)

    try:
        flow = parser.parseArgs(argv[1], argv[2:])

    except:
        parser.printHelp(argv[0])
        sys.exit()

    # Make config point to the absolute path.
    full_path = os.path.abspath(argv[0])
    last_slash_index = full_path.rfind("/")
    base_dir = full_path[0:last_slash_index]

    global CONFIG_PATH
    CONFIG_PATH = base_dir + "/" + CONFIG_PATH

    # Set debugging.
    try:
        getArg(flow, "d", "debug")
        Configurator.setDebugging(True)

    except ArgException as err:
        #         # Debugging was not specified.
        pass

    if flow[parser.KEY_MODE] != ARGS_CONFIG:
        # Load config.
        Configurator.load(CONFIG_PATH)

    if flow[parser.KEY_MODE] == ARGS_HELP:
        parser.printHelp(argv[0])
        sys.exit()

    elif flow[parser.KEY_MODE] == ARGS_COMBINE_STATISTICS:
        combineStatsFiles(flow)

    elif flow[parser.KEY_MODE] == ARGS_CONSOLE:
        id = 1
        try:
            id = getArg(flow, "id")

        except:
            pass

        heap_size = Configurator.getHeapVal()
        Neo4jHelper().setHeapsize(heap_size[0], heap_size[1])
        Neo4jHelper().startConsole(os.path.abspath(flow["in"]), str(id))

    elif flow[parser.KEY_MODE] == ARGS_SEARCH:
        startSearchMode(flow)

    elif flow[parser.KEY_MODE] == ARGS_CONTINUOUS_SEARCH:
        startSearchMode(flow, continuous=True)

    elif flow[parser.KEY_MODE] == ARGS_CONFIG:
        Configurator().setupConfig(CONFIG_PATH, base_dir,
                                   getArg(flow, "p", "path"))

    elif flow[parser.KEY_MODE] == ARGS_PRINT_STATS:
        printStats(flow)

    elif flow[parser.KEY_MODE] == ARGS_PRINT_RESULTS:
        printResults(flow)

    else:
        parser.printHelp(argv[0])
        sys.exit()
Beispiel #3
0
def main(argv):
    # Setup command line arguments.
    parser = ModeArgsParser()
    setupArgs(parser)
    
    try:
        flow = parser.parseArgs(argv[1], argv[2:])
        
    except:
        parser.printHelp(argv[0])
        sys.exit()

    # Make config point to the absolute path.
    full_path = os.path.abspath(argv[0])
    last_slash_index = full_path.rfind("/")
    base_dir = full_path[0:last_slash_index]
    
    global CONFIG_PATH
    CONFIG_PATH = base_dir + "/" + CONFIG_PATH

    # Set debugging.
    try:
        getArg(flow, "d", "debug")
        Configurator.setDebugging(True)
    
    except ArgException as err:
#         # Debugging was not specified.
        pass
        
    if flow[parser.KEY_MODE] != ARGS_CONFIG:
        # Load config.
        Configurator.load(CONFIG_PATH)

    if flow[parser.KEY_MODE] == ARGS_HELP:
        parser.printHelp(argv[0])
        sys.exit()
        
    elif flow[parser.KEY_MODE] == ARGS_COMBINE_STATISTICS:
        combineStatsFiles(flow)
    
    elif flow[parser.KEY_MODE] == ARGS_CONSOLE:
        id = 1
        try:
            id = getArg(flow, "id")
        
        except:
            pass
        
        heap_size = Configurator.getHeapVal()
        Neo4jHelper().setHeapsize(heap_size[0], heap_size[1])
        Neo4jHelper().startConsole(os.path.abspath(flow["in"]), str(id))
    
    elif flow[parser.KEY_MODE] == ARGS_SEARCH:
        startSearchMode(flow)
        
    elif flow[parser.KEY_MODE] == ARGS_CONTINUOUS_SEARCH:
        startSearchMode(flow, continuous=True)
        
    elif flow[parser.KEY_MODE] == ARGS_CONFIG:
        Configurator().setupConfig(
                                CONFIG_PATH, base_dir, getArg(flow, "p", "path")
                                )
    
    elif flow[parser.KEY_MODE] == ARGS_PRINT_STATS:
        printStats(flow)
        
    elif flow[parser.KEY_MODE] == ARGS_PRINT_RESULTS:
        printResults(flow)

    else:
        parser.printHelp(argv[0])
        sys.exit()
Beispiel #4
0
def main(argv):
    """
    Entry point of execution. Handles program arguments and
    acts accordingly. 
    """
    auth_file = "authentication"

    # Setup command line arguments.
    parser = ModeArgsParser()
    setupArgs(parser)

    flow = None
    crawler = None

    try:
        flow = parser.parseArgs(argv[1], argv[2:])

        # Check if authentication file was specified.
        if "a" in flow:
            auth_file = flow["a"]
        elif "auth" in flow:
            auth_file = flow["auth"]

    except:
        parser.printHelp(argv[0])
        sys.exit()

    # Evaluate program arguments and start program.
    if flow[parser.KEY_MODE] == ARGS_HELP:
        parser.printHelp(argv[0])

    if flow[parser.KEY_MODE] == ARGS_RATELIMIT:
        crawler = Crawler(auth_file)
        _dict = crawler.getRateLimit()
        print "Rate Limits:"
        print "core:", _dict["core"]
        print "search:", _dict["search"]

    elif flow[parser.KEY_MODE] == ARGS_CRAWL_REPOS:
        crawler = Crawler(auth_file)

        if "ds" in flow or "dontskip" in flow:
            skip = False
        else:
            skip = True

        try:
            if "f" in flow:
                _filter = flow["f"]
                _filter = convertIntoDict(_filter)

            elif "filter" in flow:
                _filter = flow["filter"]
                _filter = convertIntoDict(_filter)

            else:
                _filter = DEFAULT_REPO_FILTER

        except Exception as err:
            print err

        finally:
            crawler.crawlRepos(flow["in"], skip, _filter=_filter)

    elif flow[parser.KEY_MODE] == ARGS_EXTRACT_KEYDATA:
        if "k" in flow or "key" in flow:
            try:
                key = flow["k"]
            except:
                key = flow["key"]
            finally:
                Crawler.getKeyFromCrawlData(flow["in"], flow["out"], key)

        else:
            Crawler.getKeyFromCrawlData(flow["in"], flow["out"])

    elif flow[parser.KEY_MODE] == ARGS_EXTRACTREPOS_FILTERED:
        try:
            _filter = flow["f"]
        except:
            _filter = flow["filter"]
        finally:
            Crawler.extractReposFiltered(flow["in"], flow["out"], _filter)

    # cloning repos
    elif flow[parser.KEY_MODE] == ARGS_CLONE_REPOS:
        downloader = GitDownloader(flow["out"])

        try:
            _line = flow["l"]
        except:
            try:
                _line = flow["_line"]
            except:
                _line = 0

        delete = False
        if "d" in flow or "delete" in flow:
            delete = True

        plugin = False
        try:
            downloader.setSuccessHandler(flow["p"])
            plugin = True

        except Exception as err:
            try:
                downloader.setSuccessHandler(flow["plugin"])
                plugin = True
            except:
                pass

        if delete and not plugin:
            print(
                "A combination of -d/--delete without -p/--plugin is "
                "not allowed.")
            sys.exit()

        try:
            downloader.cloneAllFromFile(flow["in"],
                                        linenumber=_line,
                                        delete=delete)

        except OutOfScopeException as err:
            print(
                "The specified line number '%s' in parameter '-l/--line' is "
                "out of scope for file '%s'." % (_line, flow["in"]))
Beispiel #5
0
def main(argv):
    """
    Entry point of execution. Handles program arguments and
    acts accordingly. 
    """
    auth_file = "authentication"
    
    # Setup command line arguments.
    parser = ModeArgsParser()
    setupArgs(parser)
    
    flow    = None
    crawler = None
    
    try:
        flow = parser.parseArgs(argv[1], argv[2:])
        
        # Check if authentication file was specified.
        if "a" in flow:
            auth_file = flow["a"]
        elif "auth" in flow:
            auth_file = flow["auth"]
        
    except:
        parser.printHelp(argv[0])
        sys.exit()

    # Evaluate program arguments and start program.
    if flow[parser.KEY_MODE] == ARGS_HELP:
        parser.printHelp(argv[0])
    
    if flow[parser.KEY_MODE] == ARGS_RATELIMIT:
        crawler = Crawler(auth_file)
        _dict = crawler.getRateLimit()
        print "Rate Limits:"
        print "core:"  , _dict["core"]
        print "search:", _dict["search"]

    elif flow[parser.KEY_MODE] == ARGS_CRAWL_REPOS:
            crawler = Crawler(auth_file)
            
            if "ds" in flow or "dontskip" in flow:
                skip = False
            else:
                skip = True
            
            try:
                if "f" in flow:
                    _filter = flow["f"]
                    _filter = convertIntoDict(_filter)
                
                elif "filter" in flow:
                    _filter = flow["filter"]
                    _filter = convertIntoDict(_filter)
                    
                else:
                    _filter = DEFAULT_REPO_FILTER
                    
            except Exception as err:
                print err
                
            finally:
                crawler.crawlRepos(flow["in"], skip, _filter=_filter)

    elif flow[parser.KEY_MODE] == ARGS_EXTRACT_KEYDATA:
        if "k" in flow or "key" in flow:
            try:
                key = flow["k"]
            except:
                key = flow["key"]
            finally:
                Crawler.getKeyFromCrawlData(flow["in"], flow["out"], key)
            
        else:
            Crawler.getKeyFromCrawlData(flow["in"], flow["out"])
    
    elif flow[parser.KEY_MODE] == ARGS_EXTRACTREPOS_FILTERED:
        try:
            _filter = flow["f"]
        except:
            _filter = flow["filter"]
        finally:
            Crawler.extractReposFiltered(flow["in"], flow["out"], _filter)
            
    # cloning repos
    elif flow[parser.KEY_MODE] == ARGS_CLONE_REPOS:
        downloader = GitDownloader(flow["out"])
        
        try:
            _line = flow["l"]
        except:
            try:
                _line = flow["_line"]
            except:
                _line = 0
        
        delete = False
        if "d" in flow or "delete" in flow:
            delete = True

        plugin = False
        try:
            downloader.setSuccessHandler(flow["p"])
            plugin = True

        except Exception as err:
            try:
                downloader.setSuccessHandler(flow["plugin"])
                plugin = True
            except:
                pass
        
        if delete and not plugin:
            print (
                "A combination of -d/--delete without -p/--plugin is "
                "not allowed."
                )
            sys.exit()
        
        try:
            downloader.cloneAllFromFile(
                                    flow["in"], 
                                    linenumber=_line, 
                                    delete=delete
                                    )
            
        except OutOfScopeException as err:
            print (
                "The specified line number '%s' in parameter '-l/--line' is "
                "out of scope for file '%s'." % (_line, flow["in"])
                )