def testPushAndPop(self): s = self.stack s.push(Scope('A', 0)) s.push(Scope('__init__', 1)) expect = '$.A.__init__' self.assertEqual(expect, '%s' % s) s.popUntil() s.push(Scope('f', 0)) expect = '$.f' self.assertEqual(expect, '%s' % s) s.push(Scope('g', 1)) expect = '$.f.g' self.assertEqual(expect, '%s' % s)
def visit(self, node, state): stoponthis = False newobj = None # global parent scope parentscope = self.locate_scope(node, state) if not getattr(node, '_object', False): # Start traveling node when instance is created node._parent_scope = parentscope state.classes[node.name] = node stoponthis = True else: # new instance has been created # Create new Scope and push it onto the stack newscope = Scope(node, parent_scope=parentscope, is_root=True) # add builtins to scope newscope._builtins = dict( ((uv, VariableDef(uv, -1, newscope)) for uv in VariableDef.USER_VARS)) state.scopes.append(newscope) newobj = Obj(node.name, node.lineno, newscope, node._object_var, ast_node=node) # create $this var for internal method calling this_var = VariableDef('$this', node.lineno, newscope) this_var._obj_def = newobj newscope.add_var(this_var) # add ObjDef to VarDef, this way we can trace method call back to the correct instance node._object_var._obj_def = newobj node._scope = newscope state.objects[node._object_var.name] = node._object_var return newobj, stoponthis
def testEnterAndExitScope(self): scope = Scope() self.disassemble.indentTag = '\t ' line = '\t def f():' curScope = self.disassemble.getScope(line) self.assertEqual(False, self.disassemble.exitLastScope(curScope)) self.assertEqual(True, self.disassemble.enterNewScope(curScope)) self.disassemble.preScope = curScope line = 'class A_B ():' curScope = self.disassemble.getScope(line) self.assertEqual(False, self.disassemble.enterNewScope(curScope)) self.assertEqual(True, self.disassemble.exitLastScope(curScope))
def visit(self, node, state): stoponthis = False newobj = None # Methodes are not traveled untill called, this enables us to call methods from within methods method = node._parent_node._object_var._obj_def.get_method(node.name) if method: # Method object was already created, travel the children state.scopes.append(method._scope) else: # Create method so we can travel childres nodes when called parentscope = self.locate_scope(node, state) # Create new Scope and push it onto the stack newscope = Scope(node, parent_scope=parentscope, is_root=True) # node seen node._seen = True # add builtins to scope newscope._builtins = dict( ((uv, VariableDef(uv, -1, newscope)) for uv in VariableDef.USER_VARS)) # Don't trigger vulnerabilities in this scope untill code is no longer dead newscope._dead_code = True state.scopes.append(newscope) newobj = Method(node.name, node.lineno, newscope, ast_node=node) node._scope = newscope # add this method to object node._parent_node._object_var._obj_def.add_method(newobj) # Stop parsing children nodes stoponthis = True return newobj, stoponthis
def visit(self, node, state): # global parent scope parentscope = self.locate_scope(node, state) # Create new Scope and push it onto the stack newscope = Scope(node, parent_scope=parentscope, is_root=True) # add builtins to scope newscope._builtins = dict(((uv, VariableDef(uv, -1, newscope)) for uv in VariableDef.USER_VARS)) # Don't trigger vulnerabilities in this scope untill code is no longer dead newscope._dead_code = True state.scopes.append(newscope) node._scope = newscope # create Function object newobj = Function(node.name, node.lineno, newscope, ast_node=node) # Store custom function state.functions_declarations[node.name] = newobj return newobj, False
def testGetScope(self): expect = Scope() self.disassemble.indentTag = '\t ' line = '\t def f():' expect.name = 'f' expect.indentLevel = 1 self.assertEqual(True, self.disassemble.containScopeTag(line)) actual = self.disassemble.getScope(line) self.assertEqual(expect.name, actual.name) self.assertEqual(expect.indentLevel, actual.indentLevel) line = '\t class A ( object ):' expect.name = 'A' expect.indentLevel = 1 self.assertEqual(True, self.disassemble.containScopeTag(line)) actual = self.disassemble.getScope(line) self.assertEqual(expect.name, actual.name) self.assertEqual(expect.indentLevel, actual.indentLevel)
def run(self, start_urls, scopes=None): start_url = start_urls[0] self.start() start_time = time.time() scope = Scope(start_url, options=self.options.scope_options) if scopes: scope.scopes = [x.strip() for x in scopes.split(',')] self.db.start(start_url, scope.host) c = None s = None loader = None self.logger.debug("Parsing scan options") login = LoginAction(logger=self.logger.getEffectiveLevel()) pre_run = login.pre_parse(self.options) if pre_run: self.scan_cookies = dict(login.session_obj.cookies) scanoptions = [] if self.options.custom_options: scan_vars = self.options.custom_options.split(',') for v in scan_vars: opt = v.strip() scanoptions.append(opt) self.logger.debug("Enabled option %s" % opt) if self.options.scanner or self.options.allin: s = ScriptEngine(options=scanoptions, logger=self.logger.getEffectiveLevel(), database=self.db) if self.options.use_adv_scripts or self.options.allin: loader = modules.CustomModuleLoader( options=scanoptions, logger=self.logger.getEffectiveLevel(), database=self.db, scope=scope) loader.sslverify = self.options.sslverify loader.headers = login.headers loader.cookies = self.scan_cookies todo = [] c = Crawler(base_url=start_url, logger=self.logger.getEffectiveLevel()) for login_header in login.headers: c.headers[login_header] = login.headers[login_header] if self.options.use_crawler or self.options.allin: if pre_run: c.login = True # set cookies from Login module cookies = dict(login.session_obj.cookies) if cookies and len(cookies): self.logger.debug( "Setting crawler cookies from login module: %s" % str(cookies)) c.cookie.append(cookies) c.thread_count = self.thread_count c.max_urls = int(self.options.maxurls) c.scope = scope if self.options.user_agent: c.headers = {'User-Agent': self.options.user_agent} if len(start_urls) != 1: for extra_url in start_urls[1:]: c.parse_url(extra_url, extra_url) # discovery scripts, pre-run scripts and advanced modules if self.options.scanner or self.options.allin: self.logger.info("Starting filesystem discovery (pre-crawler)") new_links = s.run_fs(start_url) for newlink in new_links: c.parse_url(newlink[0], newlink[0]) if self.options.use_adv_scripts or self.options.allin: self.logger.info("Running custom scripts (pre-crawler)") links = loader.base_crawler(start_url) for link in links: self.logger.debug("Adding link %s from post scripts" % link) c.parse_url(link, link) if self.options.wl_file: wf = WebFinder(url=start_url, logger=self.logger.getEffectiveLevel(), word_list=self.options.wl_file, append=self.options.wl_ext, ok_status_codes=self.options.wl_codes, invalid_text=self.options.wl_404, threads=self.thread_count) for wf_result in wf.output: c.parse_url(wf_result, start_url) self.logger.info("Starting Crawler") c.run_scraper() self.logger.debug("Cookies set during scan: %s" % (str(c.cookie.cookies))) self.scan_cookies = c.cookie.cookies self.logger.info("Creating unique link/post data list") todo = uniquinize(c.scraped_pages) else: todo = [[start_url, None]] if self.options.driver: self.logger.info("Running GhostDriver") m = Mefjus(logger=self.logger.getEffectiveLevel(), driver_path=self.options.driver_path, use_proxy=self.options.proxy, proxy_port=self.options.proxy_port, use_https=scope.is_https, show_driver=self.options.show_driver or self.options.interactive) results = m.run(todo, interactive=self.options.interactive) for res in results: if not scope.in_scope(res[0]): self.logger.debug("IGNORE %s.. out-of-scope" % res) continue if c.get_filetype(res[0]) in c.blocked_filetypes: self.logger.debug("IGNORE %s.. bad file-type" % res) continue if res in c.scraped_pages: self.logger.debug("IGNORE %s.. exists" % res) continue else: todo.append(res) self.logger.debug("QUEUE %s" % res) self.logger.info("Creating unique link/post data list") old_num = len(todo) todo = uniquinize(todo) self.logger.debug( "WebDriver discovered %d more url/post data pairs" % (len(todo) - old_num)) scanner_obj = None if self.options.scanner or self.options.allin: self.logger.info("Starting scan sequence") if len(todo) < self.thread_count: # for performance sake self.thread_count = len(todo) scanner_obj = scanner.Scanner( logger=self.logger.getEffectiveLevel(), script_engine=s, thread_count=self.thread_count) scanner_obj.copy_engine = self.options.optimize for page in todo: url, data = page req = Request(url, data=data, agent=self.options.user_agent, headers=login.headers, cookies=self.scan_cookies) req.run() scanner_obj.queue.put(req) scanner_obj.logger.debug("Queued %s %s" % (url, data)) scanner_obj.run() post_results = [] if self.options.use_adv_scripts or self.options.allin: self.logger.info("Running post scripts") post_results = loader.run_post(todo, cookies=self.scan_cookies) cms_results = None if self.options.cms_enabled or self.options.allin: cms_loader = ext.libcms.cms_scanner_core.CustomModuleLoader( log_level=self.logger.getEffectiveLevel()) cms_results = cms_loader.run_scripts(start_url) if cms_results: for cms in cms_results: for cms_result in cms_results[cms]: self.db.put(result_type="CMS Script", script=cms, severity=0, text=cms_result) webapp_results = None if self.options.webapp_enabled or self.options.allin: webapp_loader = WebAppModuleLoader( log_level=self.logger.getEffectiveLevel()) webapp_loader.load_modules() webapp_results = webapp_loader.run_scripts( start_url, scope=scope, cookies=self.scan_cookies, headers=login.headers) if webapp_results: for webapp in webapp_results: for webapp_result in webapp_results[webapp]: self.db.put(result_type="WebApp Script", script=webapp, severity=0, text=json.dumps(webapp_result)) meta = {} if self.options.msf: monster = metamonster.MetaMonster( log_level=self.logger.getEffectiveLevel()) creds = self.options.msf_creds.split(':') monster.username = creds[0] monster.password = creds[1] monster.host = self.options.msf_host monster.port = self.options.msf_port monster.ssl = self.options.msf_ssl monster.endpoint = self.options.msf_uri monster.should_start = self.options.msf_autostart monster.connect(start_url) if monster.client and monster.client.is_working: monster.get_exploits() monster.detect() queries = monster.create_queries() monster.run_queries(queries) meta = monster.results for working in meta['working']: msf_module, msf_output = working self.db.put(result_type="Metasploit", script=msf_module, severity=3, text=json.dumps(msf_output)) scan_tree = { 'start': start_time, 'end': time.time(), 'scope': scope.host, 'starturl': start_url, 'crawled': len(c.scraped_pages) if c else 0, 'scanned': len(todo) if self.options.scanner else 0, 'results': scanner_obj.script_engine.results if scanner_obj else [], 'metasploit': meta, 'cms': cms_results, 'webapps': webapp_results, 'post': post_results if self.options.use_adv_scripts else [] } self.db.end() if self.options.outfile: with open(self.options.outfile, 'w') as f: f.write(json.dumps(scan_tree)) self.logger.info("Wrote results to %s" % self.options.outfile)
parser = argparse.ArgumentParser( description='validate hosts exported with recon-ng') parser.add_argument('scope_file', metavar='<scope file>', type=str, help='copy/paste the scope in that file') parser.add_argument('hosts_csv', metavar='<hosts csv>', type=str, help='exported hosts') parser.add_argument('--out_csv', '-o', metavar='<out csv>', default='validated_scope.csv', help='hosts validated against scope\n\ the last column is true if it is in scope, false otherwise') parser.add_argument('--verbose', '-v', action='store_true', help='verbose flag') args = parser.parse_args() scope = Scope.read_scope_from_file(filename=args.scope_file) scope_validator = ScopeValidator(scope) validated_scope = scope_validator.validate_host_csv(args.hosts_csv) if args.verbose: print('Original scope:') print(scope.ip_list) print(scope.netblock_list) print(scope.netrange_list) print(scope.hostname_list) print() print('Expended IP list:') print(scope.get_expanded_ip_list())