def log_failed_message(listid, srctype, src, msg, err): try: msgid = msg.msgid except: msgid = "<unknown>" log.error("Failed to load message (msgid %s) from %s, spec %s: %s" % (msgid.encode('us-ascii', 'replace'), srctype, src, str(str(err), 'us-ascii', 'replace'))) # We also put the data in the db. This happens in the main transaction # so if the whole script dies, it goes away... conn.cursor().execute("INSERT INTO loaderrors (listid, msgid, srctype, src, err) VALUES (%(listid)s, %(msgid)s, %(srctype)s, %(src)s, %(err)s)", { 'listid': listid, 'msgid': msgid, 'srctype': srctype, 'src': src, 'err': str(str(err), 'us-ascii', 'replace'), })
def check_http_status(scheme, host, port, task_name, task_id, tag_name): """ 检测端口的连通 状态码的有效性 压入redis,供bbscan扫描 :param scheme: :param host: :param port: :return: """ url = "%s://%s:%s" % (scheme, host, port) try: # s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # s.settimeout(5.0) # if s.connect_ex((url, int(port))) == 0: log.info('Checking Http Status Valid: %s', url) status_code, headers, content = http_request(url) if status_code: location = headers.get('Location', '') if status_code not in HTTP.IGNORE_CODE: if status_code in [301, 302 ] and location.startswith("https://"): scheme, host, port = get_hostname_port(location) status_code, headers, content = http_request(location) m = re.search('<title>(.*?)</title>', decode_text(content)) title = m.group(1) if m else '' header = get_headers(headers) banner = header + decode_text(content) rdata = { 'scheme': scheme, 'port': port, 'ip': host, 'title': title, 'status_code': status_code, "banner": banner, 'task_name': task_name, 'task_id': task_id, 'tag_name': tag_name } redis_conn.lpush("BBScan_First", json.dumps(rdata)) #压入redis,bbscan来解析扫描 return True except Exception as e: log.error('[Warning] Get http connection failed %s:%s' % (host, port), exc_info=True) return False
def reverse_host(host): """ 解析域名获取IP :param host: hostname :return: list """ ret = [] try: resolver = dns.resolver.Resolver() resolver.nameservers = NAMESERVERS answer = resolver.query(host, 'A') for i in answer: ret.append(i.address) except Exception: log.error("[Warning]: resolving hostname failed: %s", host) return return ret
def setup_callback(): global serversocket, callback_ip, callback_port, local_port, portmap_port # Don't try to map ports if we have a public IP callback_ip = callback_port = None using_portmap = False local_ip = network.get_local_ip() if not network.is_private_ip(local_ip): log.info("Host has a public IP...") callback_ip = local_ip elif config.get("port"): log.info("Host has a private IP, port specified, configure forward " + "manually...") callback_ip = network.get_public_ip() else: log.info("Host has a private IP, trying upnp port mapping...") network.portmap_setup() network.portmap_search() callback_ip = network.get_public_ip() using_portmap = True if not callback_ip: log.error("No public IP found, not registering callback.") return if not serversocket: serversocket, local_port = bind_port() log.info("Listening on port {0}...".format(local_port)) serversocket.listen(5) # max incoming calls queued if not using_portmap: callback_port = local_port else: portmap_port = network.portmap_add(portmap_port, local_port) if not portmap_port: log.error("Failed to map port, not registering callback.") return callback_port = portmap_port sync_obj.populate() for acc in sync_obj.get_bunq_accounts(): url = "https://{}:{}/bunq2ynab-autosync".format( callback_ip, callback_port) bunq_api.add_callback(acc["bunq_user_id"], acc["bunq_account_id"], "bunq2ynab-autosync", url)
def portmap_add(try_port, local_port): if not upnp: return if not try_port: try_port = local_port log.info("Adding upnp port mapping...") for i in range(0, 128): try: upnp.addportmapping(try_port, 'TCP', upnp.lanaddr, local_port, 'bynq2ynab-autosync', '') return try_port except Exception as e: if "ConflictInMappingEntry" not in str(e): log.error("Failed to map port: {}".format(e)) return log.info("Port {} is already mapped, trying next port...".format( try_port)) try_port = next_port(try_port)
def task(self, hostname): """ 执行采集器,拿到采集结果汇报给API 1. 执行所有的采集器拿到info 2. 汇报info给api :param hostname: :return: """ try: info = get_server_info(self, hostname) r1 = requests.post(url=self.asset_api, data=json.dumps(info).encode('utf-8'), headers={'Content-Type': 'application/json'}) print(r1) except Exception as e: msg = traceback.format_exc() log.error(msg) print(msg)
def log_failed_message(listid, srctype, src, msg, err): try: msgid = msg.msgid except Exception: msgid = "<unknown>" log.error("Failed to load message (msgid %s) from %s, spec %s: %s" % (msgid, srctype, src, err)) # We also put the data in the db. This happens in the main transaction # so if the whole script dies, it goes away... conn.cursor().execute( "INSERT INTO loaderrors (listid, msgid, srctype, src, err) VALUES (%(listid)s, %(msgid)s, %(srctype)s, %(src)s, %(err)s)", { 'listid': listid, 'msgid': msgid, 'srctype': srctype, 'src': src, 'err': str(err), })
def init_run(self): """ 增加301跳转的判断 :return: """ try: status_code, headers, content = http_request(self.url) location = headers.get('Location', '') if status_code in [301, 302] and location.startwith("https://"): scheme, host, port = get_hostname_port(location) self.url = "%s://%s:%s" % (scheme, host, port) status_code, headers, content = http_request(self.url) m = re.search('<title>(.*?)</title>', content) title = m.group(1) if m else '' self.header = headers self.title = title self.content = content self.get_all_uri() except: log.error("running bbscan failed", exc_info=True)
def check_port_open(host, port): """ 检测端口连通性 :param host: :param port: :return: """ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(5.0) try: if s.connect_ex((host, int(port))) == 0: log.info("Check port open %s:%s is Open", host, port) return True except Exception: log.error('[Warning] Fail to connect to %s:%s' % (host, port), exc_info=True) finally: # s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) s.close() log.info("Check port Close %s:%s is Close", host, port) return False
def _check_404_status(self): try: try: c = requests.get(self.url + '/fjkgofdjgosdjfjd-404-existence-check', verify=False, timeout=HTTP_TIMEOUT, allow_redirects=False) self._404_status = c.status_code content = c.content except: log.error('[Warning] HTTP 404 check failed <%s>', self.url, exc_info=True) self._404_status, content = -1, '' if self._404_status == 404: self.has_status_404 = True else: self.has_status_404 = False self.len_404_doc = len(content) except: log.error('[Check_404] Exception %s', self.url, exc_info=True)
def scheduler_bbscan_scan_first(): while redis_conn.llen("BBScan_First"): try: http_object = json.loads( redis_conn.lpop("BBScan_First")) # 获取第一步待扫描的内容 scheme = http_object['scheme'] ip = http_object['ip'] port = http_object['port'] header = http_object['banner'].split("\r\n\r\n\r\n")[0] content = http_object['banner'].split("\r\n\r\n\r\n")[1] status_code = http_object['status_code'] task_name = http_object['task_name'] task_id = http_object['task_id'] tag_name = http_object['tag_name'] m = re.search('<title>(.*?)</title>', content) title = m.group(1) if m else '' bbscan_parse_uri.delay(scheme, ip, port, title, content, status_code, header, task_name, task_id, tag_name) except: log.error("scheduler_bbscan_scan_first", exc_info=True)
def linux(self, handler, hostname): response = BaseResponse() try: if self.debug: ret = { 'os_platform': 'linux', 'os_version': '6.5', 'hostname': 'c1.com' } else: ret = { 'os_platform': self.os_platform(handler, hostname), 'os_version': self.os_version(handler, hostname), 'hostname': self.os_hostname(handler, hostname), } response.data = ret except Exception as e: msg = traceback.format_exc() response.status = False response.error = msg log.error(msg) return response.dict
def save_res_to_redis(self, url): try: if url == "/": for k in redis_conn_byte.scan_iter("bbscan_rules*"): rule_list = redis_conn_byte.get(k).decode('utf-8').split( ',') rule_list[0] = self.url + rule_list[0] ret = ','.join(rule_list) + ',' + str(self._404_status) + ',' + str(self.len_404_doc)+','+ \ self.task_name + ',' + self.task_id + ',' + self.tag_name # log.info("store bbscan root request to redis %s", ret) redis_conn_byte.lpush('BBScan_Second', ret) else: for k in redis_conn_byte.scan_iter("bbscan_rules_common::*"): rule_list = redis_conn_byte.get(k).decode('utf-8').split( ',') rule_list[0] = self.url + url.rstrip('/') + rule_list[0] ret = ','.join(rule_list) + ',' + str(self._404_status) + ',' + str(self.len_404_doc)+','+ \ self.task_name + ',' + self.task_id + ',' + self.tag_name # log.info("store bbscan common request to redis %s", ret) redis_conn_byte.lpush('BBScan_Second', ret) except: log.error("save to redis error", exc_info=True) pass
def get_iban_from_event(event): body_str = event.get("body") if not body_str: log.info("No request body found") return try: body = json.loads(body_str) except json.JSONDecodeError as e: log.error("Error decoding quest body as JSON: {}".format(e)) return nu = body.get("NotificationUrl", {}) category = nu.get("category") if category != "MUTATION": log.error("Category is not MUTATION but {}".format(e)) return iban = nu.get("object", {}).get("Payment", {}).get("alias", {}).get("iban") if not iban: log.error("No IBAN found in request body") return log.info("Found IBAN {} in request body".format(iban)) return iban
# We could do this "properly" by reordering operations and using ON CONFLICT, # but concurrency is not that important and this is easier... try: curs.execute("SET statement_timeout='30s'") curs.execute("SELECT pg_advisory_xact_lock(8059944559669076)") except Exception as e: print(("Failed to wait on advisory lock: %s" % e)) sys.exit(1) # Get the listid we're working on curs.execute("SELECT listid FROM lists WHERE listname=%(list)s", { 'list': opt.list }) r = curs.fetchall() if len(r) != 1: log.error("List %s not found" % opt.list) conn.close() sys.exit(1) listid = r[0][0] purges = set() if opt.directory: # Parse all files in directory for x in os.listdir(opt.directory): log.status("Parsing file %s" % x) with open(os.path.join(opt.directory, x)) as f: ap = ArchivesParserStorage() ap.parse(f) if opt.filter_msgid and not ap.is_msgid(opt.filter_msgid): continue
try: network.portmap_remove(portmap_port) except Exception as e: log.error("Error removing upnp port mapping: {}".format(e)) # ----- Main loop try: while True: try: sync_obj = sync.Sync() setup_callback() log.info("Starting periodic synchronization...") synchronize() if callback_ip and callback_port: wait_for_callback() else: log.warning("No callback, waiting for {} minutes...".format( refresh_nocallback_minutes)) time.sleep(refresh_nocallback_minutes * 60) except Exception as e: log.error("Error: {}".format(e)) log.error(traceback.format_exc()) log.error("Error occured, waiting 10 seconds.") time.sleep(10) finally: teardown_callback()
def scan_init(task_name, task_id, target, tag_name): """ 3个模块的初始化判断 1. 去除多余的空格 2. 校验是不是IP或者域名 3. 域名是否可以解析IP, IP是否可以连通 4. 如果可以连通,然后判断是否开启了端口扫描,可以的话进入到端口扫描流程 5. 判断运行的service 6. 根据运行的service,如果是http的话,进入到bbscan 7. 获取运行的service,进入到poc扫描,因为POC扫描只有两种service可识别,所以只需要判断是否是http服务就可以了。 :return: """ target = target.strip() scheme, host, port = get_hostname_port(target) target = '%s://%s:%s' % (scheme, host, port) ips, hostname = get_host_ip(target) #获取hostname和ip地址 if ips: ip = ips[0] else: return iscdn = is_cdn(ip) if len(ips) > 1: log.info("Multi ip: %s", ips) if iscdn: db.portInfo.update_one({ "task_id": task_id, "task_name": task_name }, {"$set": { "url": hostname, "cdn": true }}) log.info("CDN Check True: %s", target) ports = list(db.task.find({"task_id": task_id}, { "_id": 0, "ports": 1 }))[0]['ports'] if ports and not iscdn: _ = { "task_name": task_name, "task_id": task_id, "tag_name": tag_name, "ip": ip, "hostname": hostname, "ports": ports } log.info("Task Port Scan Begin %s", hostname) redis_conn.lpush("Task_Port_Scan", json.dumps(_)) # service = None # alive = None s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(5.0) try: if s.connect_ex((host, int(port))) == 0: log.info("Port Open %s:%s", host, port) if port == 443: target = 'https://{}'.format(hostname) service = 'https' elif port == 80: target = 'http://{}'.format(hostname) service = 'http' else: service, content = http_detect( host, port) # 端口开放的时候,判断运行的是什么服务,用来丢给POC扫描流程 if service == 'http': if 'The plain HTTP request was sent to HTTPS port' in content: service = 'https' status_code, headers, content = http_request(target) if status_code in (301, 302) and headers['Location']: _ = { "task_name": task_name, "task_id": task_id, "target": headers['Location'], "tag_name": tag_name } log.info("Http %s redirect to %s", target, headers['Location']) redis_conn.lpush("before_scan", json.dumps(_)) alive = True # content = content.eocode() if content is isinstance(content, str) else content if check_white_list(content): # 白名单储存到数据库 log.error("White List Check True") db.bbscan.update_one( { "task_id": task_id, "task_name": task_name, "tag_name": tag_name }, {"$set": { "vul_Type": "white list", "url": target }}) # return if service in ('http', 'https'): # http端口保存到资产数据库 m = re.search('<title>(.*?)</title>', content) title = m.group(1) if m else '' db.portInfo.update_one( { "task_id": task_id, "task_name": task_name, "tag_name": tag_name, "ip": ip, "port": port }, { "$set": { "server": service, "banner": content, "title": title, "hostname": hostname, "url": target } }, upsert=True) else: log.info("Port Closed %s:%s", host, port) service = False alive = False except Exception: log.error('[Warning] Fail to connect to %s:%s' % (host, port), exc_info=True) return finally: s.close() pocs = list(db.task.find({"task_id": task_id}, { "_id": 0, "pocs": 1 }))[0]['pocs'] if alive: # 主机是否存活 log.info("host %s is alive, Check Scan options", target) if pocs and service: # service表示是否开启运行了http还是unknown log.info("Begin POC Scan %s", target) _ = { "task_name": task_name, "task_id": task_id, "hostname": host, "port": port, "pocs": pocs, "tag_name": tag_name, "service": service } redis_conn.lpush("Task_Poc_Scan", json.dumps(_)) elif list( db.task.find({"task_id": task_id}, { "_id": 0, "BBScan_flag": 1 }))[0]['BBScan_flag'] and service in ('http', 'https'): _ = { "task_name": task_name, "task_id": task_id, "target": target, "tag_name": tag_name } redis_conn.lpush("BBScan_init", json.dumps(_)) else: log.info("[Warning]: No POC Selected: %s", target) pass
def bbscan(url, tag, status_to_match, content_type, content_type_no, vul_type, status_404, len_404_content, task_name, task_id, tag_name): status_to_match = int(status_to_match) status_404 = int(status_404) try: status_code, headers, content = http_request(url) cur_content_type = headers['Content-Type'] status = status_code content = decode_text(content) cur_content_length = len(content) if check_black_list(content): # 在黑名单的的url返回 return if 0 <= int(cur_content_length) <= 10: # text too short return if cur_content_type.find('image/') >= 0: # exclude image return if content_type != 'application/json' and cur_content_type.find('application/json') >= 0 and \ not url.endswith('.json'): # invalid json return if content_type and cur_content_type.find(content_type) < 0 \ or content_type_no and cur_content_type.find(content_type_no) >= 0: return # content type mismatch if tag and content.find(tag) < 0: return # tag mismatch if check_white_list(content): valid_item = True else: if status_to_match == 206 and status != 206: return if status_to_match in (200, 206) and status in (200, 206): valid_item = True elif status_to_match and status != status_to_match: return elif status in (403, 404) and status != status_to_match: return else: valid_item = True if status == status_404 and url != '/': len_doc = len(content) len_sum = int(len_404_content) + len_doc # print("bool is %s" % bool(0.4 <= float(len_doc) / len_sum <= 0.6)) if len_sum == 0 or (0.4 <= float(len_doc) / len_sum <= 0.6): return if valid_item: vul_type = vul_type.replace('_', ' ') m = re.search('<title>(.*?)</title>', content) title = m.group(1) if m else '' scheme, host, port = get_hostname_port(url) vul_url = "%s://%s:%s" % (scheme, host, port) first_find_date = datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S") # log.info(bool(db.bbscan.find_one({"task_id": task_id, "url": url}))) if db.bbscan.find_one({ "task_id": task_id, "tag_name": tag_name, "vul_url": url }): # 以task_id和url为主键查询条件 log.info("Get Vul Repeat %s", {"task_id": task_id, "url": url}) db.bbscan.update({ "task_id": task_id, "url": url }, {"$set": { "last_find_date": first_find_date }}) else: log.info("Get Vul Success %s", { "task_id": task_id, "url": url }) result = { "task_name": task_name, "task_id": task_id, "tag_name": tag_name, "vul_url": url, "url": vul_url, "vul_Type": vul_type, "status": status, "title": title, "first_find_date": first_find_date, "last_find_date": first_find_date } db.bbscan.insert(result) except TypeError: pass except KeyError: pass except: log.error("BBScan::process_request error %s", url, exc_info=True)