async def run(self): async with aiohttp.ClientSession() as session: flag = await self.check_engine_available(session, self.engine) if not flag: logger.error( "{engine_name} is not available, skipping!".format( engine_name=self.engine_name)) return logger.debug("{engine_name} is available, starting!".format( engine_name=self.engine_name)) data = {'inputurl': self.target} content = await self.get(session, self.base_url, method="POST", data=data, headers=self.headers, timeout=self.timeout, proxy=self.proxy) ret = self.check_response_errors(content) if not ret[0]: self.deal_with_errors(ret[1]) self.extract(content) logger.sysinfo("{engine} Found {num} sites".format( engine=self.engine_name, num=len(self.results['subdomain']))) logger.debug(self.engine_name + " " + str(len(self.results['subdomain'])))
def _run(domains_dic, vul_scan_flag): now_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) filename = 'srcscan_subdomain_check_' + time.strftime( "%Y%m%d_%H%M%S", time.localtime()) + '.xlsx' path = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data") if not os.path.exists(path): os.makedirs(path) for key in domains_dic.keys(): domains = list(set(domains_dic[key])) if len(domains) > 0: logger.sysinfo( "Scanning %d domains at %s." % (len(domains), time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))) for domain in domains: ret = set() ret = subdomain_scan(domain, ret, now_time) title_scan(domain, ret, now_time) if vul_scan_flag: vul_scan(domain, now_time) logger.sysinfo( "Fineshed scan %d domains at %s." % (len(domains), time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))) save(domains, path, filename, key) else: logger.error("Loading %d domains." % (len(domains))) send_smtp(path, filename)
def title_scan(domain, ret, now_time): ret = list(ret) database = Database( os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'srcscan.db')) database.connect() database.init() logger.sysinfo('Checking %d subdomains of %s.' % (len(ret), domain)) loop = asyncio.get_event_loop() thread_num = int(conf['config']['basic']['thread_num']) thread_num = thread_num if len(ret) > thread_num else thread_num tasks = [] for i in range(0, thread_num): tasks.append( asyncio.ensure_future( get_title([ret[x] for x in range(0 + i, len(ret), thread_num)]))) loop.run_until_complete(asyncio.wait(tasks)) for task in tasks: for subdomain, url, title, status, content_length in task.result(): database.update_subdomain_status(subdomain, url, title, status, content_length, now_time) database.disconnect() logger.sysinfo("Checked subdomains' status of %s." % domain)
async def run(self): async with aiohttp.ClientSession() as session: flag = await self.check_engine_available(session, self.engine) if not flag: logger.error( "{engine_name} is not available, skipping!".format( engine_name=self.engine_name)) return logger.debug("{engine_name} is available, starting!".format( engine_name=self.engine_name)) data = {'inputurl': self.target} async with session.post(self.base_url, proxy=self.proxy, data=data) as res: if res != None: try: content = await res.text() except: content = "" ret = self.check_response_errors(content) if not ret[0]: self.deal_with_errors(ret[1]) self.extract(content) logger.sysinfo("{engine} Found {num} sites".format( engine=self.engine_name, num=len(self.results['subdomain']))) logger.debug(self.engine_name + " " + str(len(self.results['subdomain'])))
def send_smtp(path, filename): try: mail_host = conf['config']['smtp']['mail_host'].strip() mail_port = int(conf['config']['smtp']['mail_port']) mail_user = conf['config']['smtp']['mail_user'] mail_pass = conf['config']['smtp']['mail_pass'] timeout = int(conf['config']['basic']['timeout']) sender = conf['config']['smtp']['sender'] receivers = conf['config']['smtp']['receivers'].split(',') except: logger.error( "Load config error: smtp, please check the config in srcscan.conf." ) return content = ''' 你好, srcscan 子域名检测结果 【%s】,请查收。 —— by srcscan ''' % (filename) message = MIMEMultipart() message['From'] = "srcscan<%s>" % sender message['To'] = ','.join(receivers) message['Subject'] = Header(filename, 'utf-8') message.attach(MIMEText(content, 'plain', 'utf-8')) with open(os.path.join(path, filename), 'rb') as f: att = MIMEText(f.read(), 'base64', 'utf-8') att["Content-Type"] = 'application/octet-stream' att.add_header("Content-Disposition", "attachment", filename=("utf-8", "", filename)) message.attach(att) n = 3 while n > 0: try: socket.setdefaulttimeout(timeout) smtpObj = smtplib.SMTP_SSL(host=mail_host) smtpObj.connect(mail_host, mail_port) smtpObj.login(mail_user, mail_pass) smtpObj.sendmail(sender, receivers, message.as_string()) logger.sysinfo("SMTP send success.") break except smtplib.SMTPException as e: logger.error("Error for SMTP: %s" % (str(e))) except socket.timeout as e: logger.error("Timeout for SMTP.") except Exception as e: print(str(e)) logger.error( "Error for SMTP, please check SMTP' config in srcscan.conf.") time.sleep(10) n -= 1
def read_domain_file(domain_file, domains_dic): domains_dic[os.path.basename(domain_file)] = [] logger.sysinfo("Loading and checking domains of file %s." % domain_file) with open(domain_file, 'r') as f: for d in f.readlines(): domain = check_domain(d) if not domain and d.strip() != '': logger.error("Error domain: %s" % d) continue domains_dic[os.path.basename(domain_file)].append(domain) return domains_dic
def update_program(): git_repository = "https://github.com/orleven/srcscan.git" success = False path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if not os.path.exists(os.path.join(path, ".git")): msg = "Have not a git repository. Please checkout the 'srcscan' repository " msg += "from GitHub (e.g. 'git clone --depth 1 https://github.com/orleven/srcscan.git srcscan')" logger.error(msg) else: msg = "Updating srcscan to the latest version from the gitHub repository." logger.sysinfo(msg) msg = "The srcscan will try to update itself using 'git' command." logger.sysinfo(msg) logger.sysinfo("Update in progress.") try: process = subprocess.Popen( "git checkout . && git pull %s HEAD" % git_repository, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=path.encode(locale.getpreferredencoding()) ) # Reference: http://blog.stastnarodina.com/honza-en/spot/python-unicodeencodeerror/ poll_process(process, True) stdout, stderr = process.communicate() success = not process.returncode except (IOError, OSError) as ex: success = False logger.error(type(ex).__name__) if success: logger.success("The latest revision '%s'" % (get_revision_number())) else: if "Not a git repository" in stderr: msg = "Not a valid git repository. Please checkout the 'orleven/srcscan' repository " msg += "from GitHub (e.g. 'git clone --depth 1 https://github.com/orleven/srcscan.git srcscan')" logger.error(msg) else: logger.error("Update could not be completed ('%s')" % re.sub(r"\W+", " ", stderr).strip()) if not success: if sys.platform == 'win32': msg = "for Windows platform it's recommended " msg += "to use a GitHub for Windows client for updating " msg += "purposes (http://windows.github.com/) or just " msg += "download the latest snapshot from " msg += "https://github.com/orleven/srcscan" else: msg = "For Linux platform it's required " msg += "to install a standard 'git' package (e.g.: 'sudo apt-get install git')" logger.sysinfo(msg)
def tocsv(datalines, path, file, key='Mysheet'): filename = os.path.join(path, file) logger.info('Export to %s...' % (filename)) if os.path.isfile(filename): book = load_workbook(filename=filename) else: book = Workbook() book.remove(book.active) if key not in book.sheetnames: ws = book.create_sheet(key) else: ws = book.get_sheet_by_name(key) i = 1 titleList = [] for line in datalines: i = i + 1 for key in line: if key not in titleList: titleList.append(key) ws.cell(row=1, column=len(titleList)).value = key try: if line[key] == None or line[key] == '': ws.cell(row=i, column=titleList.index(key) + 1).value = "" elif isinstance(line[key], int) or isinstance(line[key], str): ws.cell(row=i, column=titleList.index(key) + 1).value = line[key] elif isinstance(line[key], bytes): ws.cell(row=i, column=titleList.index(key) + 1).value = str( line[key], 'utf-8') elif isinstance(line[key], list): ws.cell(row=i, column=titleList.index(key) + 1).value = str(line[key]) elif isinstance(line[key], dict): ws.cell(row=i, column=titleList.index(key) + 1).value = str(line[key]) else: ws.cell(row=i, column=titleList.index(key) + 1).value = "Types of printing are not supported." except: ws.cell(row=i, column=titleList.index(key) + 1).value = "Some error." book.save(filename) logger.sysinfo('Exported to %s successful!' % (filename))
def run(target, vul_scan): domains_dic = {} if os.path.isdir(target): domain_file_list = glob.glob( os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), target, '*.*')) for domain_file in domain_file_list: domains_dic = read_domain_file(domain_file, domains_dic) elif os.path.isfile(target): domains_dic = read_domain_file(target, domains_dic) elif check_domain(target): logger.sysinfo("Loading and checking domain %s." % target) domains_dic[target] = [target] else: sys.exit(logger.error("Error domain: %s" % target)) _run(domains_dic, vul_scan)
def run(target): domains_dic = {} if os.path.isdir(target): domain_file_list = glob.glob(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), target, '*.*')) for domain_file in domain_file_list: domains_dic[os.path.basename(domain_file)] = [] logger.sysinfo("Loading and checking domains of file %s." % domain_file) with open(domain_file, 'r') as f: for domain in f.readlines(): domain = check_domain(domain) if not domain: logger.error("Error domain: %s" % domain) continue domains_dic[os.path.basename(domain_file)].append(domain) elif os.path.isfile(target): domains_dic[os.path.basename(target)] = [] logger.sysinfo("Loading and checking domains of file %s." % target) with open(target, 'r') as f: for domain in f.readlines(): domain = check_domain(domain) if not domain: logger.error("Error domain: %s" % domain) continue domains_dic[os.path.basename(target)].append(domain) elif check_domain(target): logger.sysinfo("Loading and checking domain %s." % target) domains_dic[target] = [target] else: sys.exit(logger.error("Error domain: %s" % target)) _run(domains_dic)
def handle(parser): args = parser.parse_args() banner() check_update(args) config_parser() domains_dic = {} # asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) if args.debug: debug = True logger.set_level(CUSTOM_LOGGING.DEBUG) nomal = args.nomal if args.help: parser.print_help() elif args.domain: domain = check_domain(args.domain) if not domain: sys.exit(logger.error("Error domain: %s" % domain)) logger.sysinfo("Loading and checking domain %s." % args.domain) domains_dic[domain]=[domain] run(domains_dic,nomal) elif args.domain_file: if os.path.isdir(args.domain_file): domain_file_list = glob.glob(os.path.join(os.path.dirname(os.path.abspath(__file__)), "domain",'*.*')) for domain_file in domain_file_list: domains_dic[os.path.basename(domain_file)] = [] logger.sysinfo("Loading and checking domains of file %s." % args.domain_file) with open(domain_file, 'r') as f: for domain in f.readlines(): domain = check_domain(domain) if not domain: logger.error("Error domain: %s" % domain) continue domains_dic[os.path.basename(domain_file)].append(domain) run(domains_dic,nomal) elif os.path.isfile(args.domain_file): domains_dic[os.path.basename(args.domain_file)] = [] logger.sysinfo("Loading and checking domains of file %s." % args.domain_file) with open(args.domain_file, 'r') as f: for domain in f.readlines(): domain = check_domain(domain) if not domain: logger.error("Error domain: %s" % domain) continue domains_dic[os.path.basename(args.domain_file)].append(domain) run(domains_dic,nomal) else: logger.sysinfo("Error for domain file, please check the file %s." % args.domain_file) else: parser.print_help()
def subdomain_scan(domain, ret, now_time): database = Database( os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'srcscan.db')) database.connect() database.init() logger.sysinfo("Scanning domain %s." % domain) _engines = [_(domain) for _ in engines.values()] loop = asyncio.get_event_loop() if debug: loop.set_debug(True) for task in [asyncio.ensure_future(_engine.run()) for _engine in _engines]: loop.run_until_complete(task) # loop.close() for _engine in _engines: logger.sysinfo("{engine} Found {num} sites".format( engine=_engine.engine_name, num=len(_engine.results['subdomain']))) ret.update(_engine.results['subdomain']) logger.sysinfo("Found %d subdomains of %s." % (len(ret), domain)) for subdomain in ret: database.insert_subdomain(subdomain, None, None, 0, 0, now_time, domain) database.disconnect() return ret
def vul_scan(domain, now_time): datas = [] database = Database( os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'srcscan.db')) database.connect() database.init() logger.sysinfo("Scaning vul for: %s " % (domain)) for _row in database.select_mondomain(domain): data = { "subdomain": _row[0], "url": _row[1], "title": _row[2], "status": _row[3], "len": _row[4], "update_time": _row[5], "domain": _row[6] } datas.append(data) for data in datas: if data['status'] != 0: logger.sysinfo("Scaning vul for %s." % (data['url'])) crawlergo_scan(data['url'], data['domain'], now_time, database) logger.sysinfo("Scaned vul for: %s " % (domain)) database.disconnect()
def crawlergo_scan(url, domain, now_time, database): cmd = [ conf['config']['crawlergo']['crawlergo_path'], "-c", conf['config']['crawlergo']['chrome_path'], "-t", "20", "-f", "smart", "--fuzz-path", "--output-mode", "json", url ] rsp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = rsp.communicate() try: result = json.loads(output.decode().split("--[Mission Complete]--")[1]) _req_list = result["req_list"] sub_domain_list = result["sub_domain_list"] except: return req_list = [] for req in _req_list: if url.strip('http://').strip('https://').rstrip('/') in req['url']: req_list.append(req) else: logger.sysinfo("Skip %s url by %s." % (req['url'], url)) logger.sysinfo("Found %d url by %s." % (len(req_list), url)) logger.sysinfo("Found %d subdomains by %s." % (len(sub_domain_list), url)) for subdomain in sub_domain_list: database.insert_subdomain(subdomain, None, None, 0, 0, now_time, domain) # logger.sysinfo('Checking %d subdomains by %s.' % (len(sub_domain_list), url)) # for subdomain, url, title, status, content_length in curl.run(): # database.update_subdomain_status(subdomain, url, title, status, content_length, now_time) # logger.sysinfo("Checked subdomains' status by %s." % url) loop = asyncio.get_event_loop() thread_num = int(conf['config']['basic']['thread_num']) thread_num = thread_num if len(req_list) > thread_num else thread_num tasks = [] for i in range(0, thread_num): tasks.append( asyncio.ensure_future( go_request([ req_list[x] for x in range(0 + i, len(req_list), thread_num) ], url))) loop.run_until_complete(asyncio.wait(tasks))
def poll_process(process, suppress_errors=False): """ Checks for process status (prints . if still running) """ while True: time.sleep(1) returncode = process.poll() if returncode is not None: if not suppress_errors: if returncode == 0: logger.sysinfo(" done\n") elif returncode < 0: logger.sysinfo(" process terminated by signal %d\n" % returncode) elif returncode > 0: logger.sysinfo(" quit unexpectedly with return code %d\n" % returncode) break
def _run(domains_dic): database = Database(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'submon.db')) database.connect() database.init() now_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) filename = 'SubMon_subdomain_check_' + time.strftime("%Y%m%d_%H%M%S", time.localtime()) + '.xlsx' path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data") if not os.path.exists(path): os.makedirs(path) for key in domains_dic.keys(): domains = list(set(domains_dic[key])) if len(domains) > 0: logger.sysinfo("Scanning %d domains at %s." % (len(domains), time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))) for domain in domains: logger.sysinfo("Scanning domain %s." % domain) _engines = [_(domain) for _ in engines.values()] loop = asyncio.get_event_loop() if debug: loop.set_debug(True) for task in [asyncio.ensure_future(_engine.run()) for _engine in _engines ]: loop.run_until_complete(task) # loop.close() ret = set() for _engine in _engines: logger.sysinfo("{engine} Found {num} sites".format(engine=_engine.engine_name, num=len(_engine.results['subdomain']))) ret.update(_engine.results['subdomain']) logger.sysinfo("Found %d subdomains of %s." % (len(ret),domain)) for subdomain in ret: database.insert_subdomain(subdomain,None,None,0,0,now_time,domain) logger.sysinfo('Checking %d subdomains of %s.' % (len(ret),domain)) curl = Curl() curl.load_targets(ret) for subdomain,url,title,status,content_length in curl.run(): database.update_subdomain_status(subdomain,url,title,status,content_length,now_time) logger.sysinfo("Checked subdomains' status of %s." % domain) datas = [] for domain in domains: for _row in database.select_mondomain(domain): data = { "subdomain": _row[0], "url": _row[1], "title": _row[2], "status": _row[3], "len": _row[4], "update_time" : _row[5], "domain": _row[6] } datas.append(data) tocsv(datas, path,filename,key) logger.sysinfo("Fineshed scan %d domains at %s." % (len(domains), time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))) else: logger.error("Loading %d domains." % (len(domains))) send_smtp(path, filename) database.disconnect() print() print()