def test_alive(ip_str="180.188.250.54", begin=45, end=60, interval=2): start_time = time.time() test_array = {} for i in range(begin, end, interval): sslsock, _, _ = connect_ssl(ip_str) try: result = test_app_check(sslsock, ip_str) except: break test_array[i] = {} test_array[i]["ssl"] = sslsock test_array[i]["start_time"] = time.time() time.sleep(interval) time_now = time.time() time.sleep(end - (time_now - start_time)) for i in test_array: stat = test_array[i] sslsock = stat["ssl"] time_now = time.time() try: result = test_app_check(sslsock, ip_str) xlog.info("time alive:%d", time_now - stat["start_time"]) except: xlog.info("time alive fail:%d", time_now - stat["start_time"]) continue
def create_ca(): key = OpenSSL.crypto.PKey() key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) req = OpenSSL.crypto.X509Req() subj = req.get_subject() subj.countryName = 'CN' subj.stateOrProvinceName = 'Internet' subj.localityName = 'Cernet' subj.organizationName = CertUtil.ca_vendor subj.organizationalUnitName = '%s Root' % CertUtil.ca_vendor subj.commonName = '%s XX-Net' % CertUtil.ca_vendor #TODO: here should be GoAgent req.set_pubkey(key) req.sign(key, CertUtil.ca_digest) ca = OpenSSL.crypto.X509() ca.set_version(2) ca.set_serial_number(0) ca.gmtime_adj_notBefore(0) ca.gmtime_adj_notAfter(24 * 60 * 60 * 3652) ca.set_issuer(req.get_subject()) ca.set_subject(req.get_subject()) ca.set_pubkey(req.get_pubkey()) ca.add_extensions([ OpenSSL.crypto.X509Extension('basicConstraints', False, 'CA:TRUE', ca, ca) ]) ca.sign(key, CertUtil.ca_digest) #logging.debug("CA key:%s", key) xlog.info("create ca") return key, ca
def scan_ip_worker(self): while self.searching_thread_count <= self.scan_ip_thread_num: if not connect_control.allow_scan(): time.sleep(10) continue try: time.sleep(1) ip_int = ip_range.get_ip() ip_str = ip_utils.ip_num_to_string(ip_int) if self.is_bad_ip(ip_str): continue result = check_ip.test_gws(ip_str) if not result: continue if self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type): #logging.info("add %s CN:%s type:%s time:%d gws:%d ", ip_str, # result.domain, result.server_type, result.handshake_time, len(self.gws_ip_list)) xlog.info("scan_ip add ip:%s time:%d", ip_str, result.handshake_time) scan_ip_log.info("Add %s time:%d CN:%s type:%s", ip_str, result.handshake_time, result.domain, result.server_type) self.remove_slowest_ip() self.save_ip_list() except check_ip.HoneypotError as e: self.report_bad_ip(ip_str) connect_control.fall_into_honeypot() continue except Exception as e: xlog.exception("google_ip.runJob fail:%s", e) self.ncount_lock.acquire() self.searching_thread_count -= 1 self.ncount_lock.release() xlog.info("scan_ip_worker exit")
def test2(self): work_ciphers = ["AES128-SHA"] for cipher in self.cipher_list: if cipher in work_ciphers: continue else: work_ciphers.append(cipher) xlog.debug("%s", cipher) cipher_suites = (work_ciphers) openssl_context = SSLConnection.context_builder(ca_certs=g_cacertfile, cipher_suites=cipher_suites) try: ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context) server_type = test_server_type(ssl, self.ip) xlog.debug("%s", server_type) if "gws" not in server_type: work_ciphers.remove(cipher) except Exception as e: xlog.warn("err:%s", e) try: work_ciphers.remove(cipher) except: pass work_str = "" for cipher in work_ciphers: work_str += cipher + ":" xlog.info("work ciphers:%s", work_str)
def context_builder(ca_certs=None, cipher_suites=('ALL:!RC4-SHA:!ECDHE-RSA-RC4-SHA:!ECDHE-RSA-AES128-GCM-SHA256:!AES128-GCM-SHA256',)): # 'ALL', '!aNULL', '!eNULL' global ssl_version if not ssl_version: if hasattr(OpenSSL.SSL, "TLSv1_2_METHOD"): ssl_version = "TLSv1_2" elif hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"): ssl_version = "TLSv1_1" elif hasattr(OpenSSL.SSL, "TLSv1_METHOD"): ssl_version = "TLSv1" else: ssl_version = "SSLv23" if sys.platform == "darwin": ssl_version = "TLSv1" # freenas openssl support fix from twitter user "himanzero" # https://twitter.com/himanzero/status/645231724318748672 if sys.platform == "freebsd9": ssl_version = "TLSv1" xlog.info("SSL use version:%s", ssl_version) protocol_version = getattr(OpenSSL.SSL, '%s_METHOD' % ssl_version) ssl_context = OpenSSL.SSL.Context(protocol_version) if ca_certs: ssl_context.load_verify_locations(os.path.abspath(ca_certs)) ssl_context.set_verify(OpenSSL.SSL.VERIFY_PEER, lambda c, x, e, d, ok: ok) else: ssl_context.set_verify(OpenSSL.SSL.VERIFY_NONE, lambda c, x, e, d, ok: ok) ssl_context.set_cipher_list(':'.join(cipher_suites)) return ssl_context
def check_win10(): if sys.platform != "win32": return False import ctypes class OSVERSIONINFOEXW(ctypes.Structure): _fields_ = [ ("dwOSVersionInfoSize", ctypes.c_ulong), ("dwMajorVersion", ctypes.c_ulong), ("dwMinorVersion", ctypes.c_ulong), ("dwBuildNumber", ctypes.c_ulong), ("dwPlatformId", ctypes.c_ulong), ("szCSDVersion", ctypes.c_wchar * 128), ("wServicePackMajor", ctypes.c_ushort), ("wServicePackMinor", ctypes.c_ushort), ("wSuiteMask", ctypes.c_ushort), ("wProductType", ctypes.c_byte), ("wReserved", ctypes.c_byte), ] os_version = OSVERSIONINFOEXW() os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version) retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version)) if retcode != 0: xlog.warn("Failed to get win32 OS version") return False if os_version.dwMajorVersion == 10: xlog.info("detect Win10, enable connect concurent control.") return True return False
def load_openssl(): global loaded, libcrypto, buf from ctypes.util import find_library for p in ('crypto', 'eay32', 'libeay32'): libcrypto_path = find_library(p) if libcrypto_path: break else: raise Exception('libcrypto(OpenSSL) not found') xlog.info('loading libcrypto from %s', libcrypto_path) libcrypto = CDLL(libcrypto_path) libcrypto.EVP_get_cipherbyname.restype = c_void_p libcrypto.EVP_CIPHER_CTX_new.restype = c_void_p libcrypto.EVP_CipherInit_ex.argtypes = (c_void_p, c_void_p, c_char_p, c_char_p, c_char_p, c_int) libcrypto.EVP_CipherUpdate.argtypes = (c_void_p, c_void_p, c_void_p, c_char_p, c_int) libcrypto.EVP_CIPHER_CTX_cleanup.argtypes = (c_void_p,) libcrypto.EVP_CIPHER_CTX_free.argtypes = (c_void_p,) if hasattr(libcrypto, 'OpenSSL_add_all_ciphers'): libcrypto.OpenSSL_add_all_ciphers() buf = create_string_buffer(buf_size) loaded = True
def test2(self): work_ciphers = ["AES128-SHA"] for cipher in self.cipher_list: if cipher in work_ciphers: continue else: work_ciphers.append(cipher) xlog.debug("%s", cipher) cipher_suites = (work_ciphers) openssl_context = SSLConnection.context_builder( ca_certs=g_cacertfile, cipher_suites=cipher_suites) try: ssl, _, _ = connect_ssl(self.ip, openssl_context=openssl_context) server_type = test_server_type(ssl, self.ip) xlog.debug("%s", server_type) if "gws" not in server_type: work_ciphers.remove(cipher) except Exception as e: xlog.warn("err:%s", e) try: work_ciphers.remove(cipher) except: pass work_str = "" for cipher in work_ciphers: work_str += cipher + ":" xlog.info("work ciphers:%s", work_str)
def load_ip(self): if os.path.isfile(self.good_ip_file): file_path = self.good_ip_file else: file_path = self.default_good_ip_file with open(file_path, "r") as fd: lines = fd.readlines() for line in lines: try: if line.startswith("#"): continue str_l = line.split(" ") if len(str_l) < 4: xlog.warning("line err: %s", line) continue ip_str = str_l[0] domain = str_l[1] server = str_l[2] handshake_time = int(str_l[3]) if len(str_l) > 4: fail_times = int(str_l[4]) else: fail_times = 0 # logging.info("load ip: %s time:%d domain:%s server:%s", ip_str, handshake_time, domain, server) self.add_ip(ip_str, handshake_time, domain, server, fail_times) except Exception as e: xlog.exception("load_ip line:%s err:%s", line, e) xlog.info("load google ip_list num:%d, gws num:%d", len(self.ip_dict), len(self.gws_ip_list)) self.try_sort_gws_ip(force=True)
def remove_slowest_ip(self): if len(self.gws_ip_list) <= self.max_good_ip_num: return self.try_sort_ip_by_handshake_time(force=True) self.ip_lock.acquire() try: ip_num = len(self.gws_ip_list) while ip_num > self.max_good_ip_num: ip_str = self.gws_ip_list[ip_num - 1] property = self.ip_dict[ip_str] server = property['server'] handshake_time = property['handshake_time'] xlog.info("remove_slowest_ip:%s handshake_time:%d", ip_str, handshake_time) del self.ip_dict[ip_str] if 'gws' in server and ip_str in self.gws_ip_list: self.gws_ip_list.remove(ip_str) ip_num -= 1 except Exception as e: xlog.exception("remove_slowest_ip err:%s", e) finally: self.ip_lock.release()
def check_win10(): if sys.platform != "win32": return False import ctypes class OSVERSIONINFOEXW(ctypes.Structure): _fields_ = [('dwOSVersionInfoSize', ctypes.c_ulong), ('dwMajorVersion', ctypes.c_ulong), ('dwMinorVersion', ctypes.c_ulong), ('dwBuildNumber', ctypes.c_ulong), ('dwPlatformId', ctypes.c_ulong), ('szCSDVersion', ctypes.c_wchar*128), ('wServicePackMajor', ctypes.c_ushort), ('wServicePackMinor', ctypes.c_ushort), ('wSuiteMask', ctypes.c_ushort), ('wProductType', ctypes.c_byte), ('wReserved', ctypes.c_byte)] os_version = OSVERSIONINFOEXW() os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version) retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version)) if retcode != 0: xlog.warn("Failed to get win32 OS version") return False if os_version.dwMajorVersion == 10: xlog.info("detect Win10, enable connect concurent control.") return True return False
def do_POST(self): try: refer = self.headers.getheader('Referer') netloc = urlparse.urlparse(refer).netloc if not netloc.startswith("127.0.0.1") and not netloc.startswitch("localhost"): xlog.warn("web control ref:%s refuse", netloc) return except: pass xlog.debug ('GAEProxy web_control %s %s %s ', self.address_string(), self.command, self.path) try: ctype, pdict = cgi.parse_header(self.headers.getheader('content-type')) if ctype == 'multipart/form-data': self.postvars = cgi.parse_multipart(self.rfile, pdict) elif ctype == 'application/x-www-form-urlencoded': length = int(self.headers.getheader('content-length')) self.postvars = urlparse.parse_qs(self.rfile.read(length), keep_blank_values=1) else: self.postvars = {} except: self.postvars = {} path = urlparse.urlparse(self.path).path if path == '/deploy': return self.req_deploy_handler() elif path == "/config": return self.req_config_handler() elif path == "/scan_ip": return self.req_scan_ip_handler() elif path.startswith("/importip"): return self.req_importip_handler() else: self.wfile.write(b'HTTP/1.1 404\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n404 Not Found') xlog.info('%s "%s %s HTTP/1.1" 404 -', self.address_string(), self.command, self.path)
def scan_ip_worker(self): while self.searching_thread_count <= self.scan_ip_thread_num and connect_control.keep_running: if not connect_control.allow_scan(): time.sleep(10) continue try: time.sleep(1) ip_int = ip_range.get_ip() ip_str = ip_utils.ip_num_to_string(ip_int) if ip_str in self.ip_dict: continue connect_control.start_connect_register() result = check_ip.test_gae(ip_str) connect_control.end_connect_register() if not result: continue if self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type): #logging.info("add %s CN:%s type:%s time:%d gws:%d ", ip_str, # result.domain, result.server_type, result.handshake_time, len(self.gws_ip_list)) xlog.info("scan_ip add ip:%s time:%d", ip_str, result.handshake_time) scan_ip_log.info("Add %s time:%d CN:%s type:%s", ip_str, result.handshake_time, result.domain, result.server_type) self.remove_slowest_ip() self.save_ip_list() except Exception as e: xlog.exception("google_ip.runJob fail:%s", e) self.ncount_lock.acquire() self.searching_thread_count -= 1 self.ncount_lock.release() xlog.info("scan_ip_worker exit")
def test_alive(ip_str="74.125.96.107", begin=50, end=60, interval=2): test_array = {} for i in range(begin, end, interval): sslsock, _, _ = connect_ssl(ip_str) try: result = test_app_check(sslsock, ip_str) except: break test_array[i] = {} test_array[i]["ssl"] = sslsock test_array[i]["start_time"] = time.time() time.sleep(interval) time.sleep(begin) for i in test_array: stat = test_array[i] sslsock = stat["ssl"] time_now = time.time() try: result = test_app_check(sslsock, ip_str) xlog.info("time alive:%d", time_now - stat["start_time"]) except: xlog.info("time alive fail") break
def remove_ip_process(self): try: while connect_control.keep_running: try: ip_str = self.to_remove_ip_list.get_nowait() except: break result = check_ip.test(ip_str) if result and result.appspot_ok: self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type) xlog.debug("remove ip process, restore ip:%s", ip_str) continue if not check_ip.network_is_ok(): self.to_remove_ip_list.put(ip_str) xlog.warn( "network is unreachable. check your network connection." ) return xlog.info("real remove ip:%s ", ip_str) self.iplist_need_save = 1 finally: self.remove_ip_thread_num_lock.acquire() self.remove_ip_thread_num -= 1 self.remove_ip_thread_num_lock.release()
def check_all_exist_ip(): good_ip_file_name = "good_ip.txt" good_ip_file = os.path.abspath( os.path.join(config.DATA_PATH, good_ip_file_name)) if not os.path.isfile(good_ip_file): print "open file ", good_ip_file_name, " fail." return with open(good_ip_file, "r") as fd: lines = fd.readlines() for line in lines: try: str_l = line.split(' ') if len(str_l) != 4: xlog.warning("line err: %s", line) continue ip_str = str_l[0] domain = str_l[1] server = str_l[2] handshake_time = int(str_l[3]) xlog.info("test ip: %s time:%d domain:%s server:%s", ip_str, handshake_time, domain, server) #test_with_app(ip_str) test_gws(ip_str) #self.add_ip(ip_str, handshake_time, domain, server) except Exception as e: xlog.exception("load_ip line:%s err:%s", line, e)
def load_ip(self): if os.path.isfile(self.good_ip_file): file_path = self.good_ip_file else: file_path = self.default_good_ip_file with open(file_path, "r") as fd: lines = fd.readlines() for line in lines: try: str_l = line.split(' ') if len(str_l) < 4: xlog.warning("line err: %s", line) continue ip_str = str_l[0] domain = str_l[1] server = str_l[2] handshake_time = int(str_l[3]) if len(str_l) > 4: fail_times = int(str_l[4]) else: fail_times = 0 #logging.info("load ip: %s time:%d domain:%s server:%s", ip_str, handshake_time, domain, server) self.add_ip(ip_str, handshake_time, domain, server, fail_times) except Exception as e: xlog.exception("load_ip line:%s err:%s", line, e) xlog.info("load google ip_list num:%d, gws num:%d", len(self.ip_dict), len(self.gws_ip_list)) self.try_sort_gws_ip(force=True)
def load_tasks(): tasks = [] intervals = [] import imp tasks_path = config.tasks_path for fileName in os.listdir(tasks_path): if fileName.lower().endswith('.py'): xlog.info('start load %s' % fileName) try: with open(os.path.join(tasks_path, fileName), 'rb') as fpy: task = imp.load_source('Auto-tasks_%d_%s' % (len(tasks), fileName), tasks_path, fpy) name = task.name interval = task.run_interval task.init() tasks.append(task) intervals.append(interval) xlog.info('load task %s success.' % name) except: xlog.warn('load %s fail.' % fileName) del imp return (tasks, intervals)
def main(data_path="."): xlog.info("listen http on 8880") httpd = HTTPServer(('', 8880), TestHttpServer, data_path) httpd.start() while True: time.sleep(10)
def remove_slowest_ip(self): if len(self.gws_ip_list) <= self.max_good_ip_num: return self.try_sort_gws_ip(force=True) self.ip_lock.acquire() try: ip_num = len(self.gws_ip_list) while ip_num > self.max_good_ip_num: ip_str = self.gws_ip_list[ip_num - 1] property = self.ip_dict[ip_str] server = property["server"] fails = property["fail_times"] handshake_time = property["handshake_time"] xlog.info("remove_slowest_ip:%s handshake_time:%d, fails:%d", ip_str, handshake_time, fails) del self.ip_dict[ip_str] if "gws" in server and ip_str in self.gws_ip_list: self.gws_ip_list.remove(ip_str) ip_num -= 1 except Exception as e: xlog.exception("remove_slowest_ip err:%s", e) finally: self.ip_lock.release()
def test(ip_str, loop=1): xlog.info("==>%s", ip_str) check = Check_frame(ip_str, check_cert=False) for i in range(loop): result = check.check(callback=test_app_head) if not result: if "gws" in check.result.server_type: xlog.warn("ip:%s server_type:%s but appengine check fail.", ip_str, check.result.server_type) xlog.warn("check fail") #continue else: xlog.debug("=======app check ok: %s", ip_str) check.result.appspot_ok = result result = check.check(callback=test_server_type, check_ca=True) if not result: xlog.debug("test server type fail") continue check.result.server_type = result xlog.info("========== %s type:%s domain:%s handshake:%d", ip_str, check.result.server_type, check.result.domain, check.result.handshake_time) return check.result
def create_ca(): key = OpenSSL.crypto.PKey() key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) req = OpenSSL.crypto.X509Req() subj = req.get_subject() subj.countryName = 'CN' subj.stateOrProvinceName = 'Internet' subj.localityName = 'Cernet' subj.organizationName = CertUtil.ca_vendor subj.organizationalUnitName = '%s Root' % CertUtil.ca_vendor subj.commonName = '%s XX-Net' % CertUtil.ca_vendor #TODO: here should be GoAgent req.set_pubkey(key) req.sign(key, CertUtil.ca_digest) ca = OpenSSL.crypto.X509() ca.set_version(2) ca.set_serial_number(0) ca.gmtime_adj_notBefore(0) ca.gmtime_adj_notAfter(24 * 60 * 60 * 3652) ca.set_issuer(req.get_subject()) ca.set_subject(req.get_subject()) ca.set_pubkey(req.get_pubkey()) ca.add_extensions([ OpenSSL.crypto.X509Extension( 'basicConstraints', False, 'CA:TRUE', ca, ca) ]) ca.sign(key, CertUtil.ca_digest) #logging.debug("CA key:%s", key) xlog.info("create ca") return key, ca
def remove_ip_process(self): try: while True: try: ip_str = self.to_remove_ip_list.get_nowait() except: break result = check_ip.test(ip_str) if result and result.appspot_ok: self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type) xlog.debug("remove ip process, restore ip:%s", ip_str) continue if not check_ip.network_is_ok(): self.to_remove_ip_list.put(ip_str) xlog.warn("network is unreachable. check your network connection.") return xlog.info("real remove ip:%s ", ip_str) self.iplist_need_save = 1 finally: self.remove_ip_thread_num_lock.acquire() self.remove_ip_thread_num -= 1 self.remove_ip_thread_num_lock.release()
def import_mac_ca(common_name, certfile): commonname = "GoAgent XX-Net" #TODO: need check again ca_hash = CertUtil.ca_thumbprint.replace(':', '') def get_exist_ca_sha1(): args = [ 'security', 'find-certificate', '-Z', '-a', '-c', commonname ] output = subprocess.check_output(args) for line in output.splitlines(True): if len(line) == 53 and line.startswith("SHA-1 hash:"): sha1_hash = line[12:52] return sha1_hash exist_ca_sha1 = get_exist_ca_sha1() if exist_ca_sha1 == ca_hash: xlog.info("GoAgent CA exist") return import_command = 'security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ../../data/gae_proxy/CA.crt' # % certfile.decode('utf-8') if exist_ca_sha1: delete_ca_command = 'security delete-certificate -Z %s' % exist_ca_sha1 exec_command = "%s;%s" % (delete_ca_command, import_command) else: exec_command = import_command admin_command = """osascript -e 'do shell script "%s" with administrator privileges' """ % exec_command cmd = admin_command.encode('utf-8') xlog.info("try auto import CA command:%s", cmd) os.system(cmd)
def report_connect_fail(self, ip_str, force_remove=False): self.ip_lock.acquire() try: time_now = time.time() if not ip_str in self.ip_dict: return self.ip_dict[ip_str]["links"] -= 1 # ignore if system network is disconnected. if not force_remove: if not check_ip.network_is_ok(): xlog.debug("report_connect_fail network fail") # connect_control.fall_into_honeypot() return fail_time = self.ip_dict[ip_str]["fail_time"] if not force_remove and time_now - fail_time < 1: xlog.debug("fail time too near") return # increase handshake_time to make it can be used in lower probability self.ip_dict[ip_str]["handshake_time"] += 300 if self.ip_dict[ip_str]["fail_times"] == 0: self.good_ip_num -= 1 self.ip_dict[ip_str]["fail_times"] += 1 self.append_ip_history(ip_str, "fail") self.ip_dict[ip_str]["fail_time"] = time_now if force_remove or self.ip_dict[ip_str]["fail_times"] >= 50: property = self.ip_dict[ip_str] server = property["server"] del self.ip_dict[ip_str] if "gws" in server and ip_str in self.gws_ip_list: self.gws_ip_list.remove(ip_str) if not force_remove: self.to_remove_ip_list.put(ip_str) self.try_remove_thread() xlog.info( "remove ip tmp:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list) ) else: xlog.info( "remove ip:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list) ) if self.good_ip_num > len(self.ip_dict): self.good_ip_num = len(self.ip_dict) self.iplist_need_save = 1 except Exception as e: xlog.exception("set_ip err:%s", e) finally: self.ip_lock.release() if not self.is_ip_enough(): self.search_more_google_ip()
def import_mac_ca(common_name, certfile): commonname = "GoAgent XX-Net" #TODO: need check again ca_hash = CertUtil.ca_thumbprint.replace(':', '') def get_exist_ca_sha1(): args = ['security', 'find-certificate', '-Z', '-a', '-c', commonname] output = subprocess.check_output(args) for line in output.splitlines(True): if len(line) == 53 and line.startswith("SHA-1 hash:"): sha1_hash = line[12:52] return sha1_hash exist_ca_sha1 = get_exist_ca_sha1() if exist_ca_sha1 == ca_hash: xlog.info("GoAgent CA exist") return import_command = 'security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ../../data/gae_proxy/CA.crt'# % certfile.decode('utf-8') if exist_ca_sha1: delete_ca_command = 'security delete-certificate -Z %s' % exist_ca_sha1 exec_command = "%s;%s" % (delete_ca_command, import_command) else: exec_command = import_command admin_command = """osascript -e 'do shell script "%s" with administrator privileges' """ % exec_command cmd = admin_command.encode('utf-8') xlog.info("try auto import CA command:%s", cmd) os.system(cmd)
def load_openssl(): global loaded, libcrypto, buf from ctypes.util import find_library for p in ('crypto', 'eay32', 'libeay32'): libcrypto_path = find_library(p) if libcrypto_path: break else: raise Exception('libcrypto(OpenSSL) not found') xlog.info('loading libcrypto from %s', libcrypto_path) libcrypto = CDLL(libcrypto_path) libcrypto.EVP_get_cipherbyname.restype = c_void_p libcrypto.EVP_CIPHER_CTX_new.restype = c_void_p libcrypto.EVP_CipherInit_ex.argtypes = (c_void_p, c_void_p, c_char_p, c_char_p, c_char_p, c_int) libcrypto.EVP_CipherUpdate.argtypes = (c_void_p, c_void_p, c_void_p, c_char_p, c_int) libcrypto.EVP_CIPHER_CTX_cleanup.argtypes = (c_void_p, ) libcrypto.EVP_CIPHER_CTX_free.argtypes = (c_void_p, ) if hasattr(libcrypto, 'OpenSSL_add_all_ciphers'): libcrypto.OpenSSL_add_all_ciphers() buf = create_string_buffer(buf_size) loaded = True
def __init__(self, address, handler, args=()): self.socket = None self.running = True self.server_address = address self.handler = handler self.args = args self.init_socket() xlog.info("server %s:%d started.", address[0], address[1])
def report_connect_fail(self, ip_str, force_remove=False): self.ip_lock.acquire() try: time_now = time.time() if not ip_str in self.ip_dict: return self.ip_dict[ip_str]['links'] -= 1 # ignore if system network is disconnected. if not force_remove: if not check_ip.network_is_ok(): xlog.debug("report_connect_fail network fail") #connect_control.fall_into_honeypot() return fail_time = self.ip_dict[ip_str]["fail_time"] if not force_remove and time_now - fail_time < 1: xlog.debug("fail time too near") return # increase handshake_time to make it can be used in lower probability self.ip_dict[ip_str]['handshake_time'] += 300 if self.ip_dict[ip_str]['fail_times'] == 0: self.good_ip_num -= 1 self.ip_dict[ip_str]['fail_times'] += 1 self.append_ip_history(ip_str, "fail") self.ip_dict[ip_str]["fail_time"] = time_now if force_remove or self.ip_dict[ip_str]['fail_times'] >= 50: property = self.ip_dict[ip_str] server = property['server'] del self.ip_dict[ip_str] if 'gws' in server and ip_str in self.gws_ip_list: self.gws_ip_list.remove(ip_str) if not force_remove: self.to_remove_ip_list.put(ip_str) self.try_remove_thread() xlog.info("remove ip tmp:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list)) else: xlog.info("remove ip:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list)) if self.good_ip_num > len(self.ip_dict): self.good_ip_num = len(self.ip_dict) self.iplist_need_save = 1 except Exception as e: xlog.exception("set_ip err:%s", e) finally: self.ip_lock.release() if not self.is_ip_enough(): self.search_more_google_ip()
def import_debian_ca(common_name, ca_file): def get_debian_ca_sha1(nss_path): commonname = "GoAgent XX-Net - GoAgent" #TODO: here should be GoAgent - XX-Net cmd = [ 'certutil', '-L', '-d', 'sql:%s' % nss_path, '-n', commonname ] lines = get_cmd_out(cmd) get_sha1_title = False sha1 = "" for line in lines: if line.endswith("Fingerprint (SHA1):\n"): get_sha1_title = True continue if get_sha1_title: sha1 = line break sha1 = sha1.replace(' ', '').replace(':', '').replace('\n', '') if len(sha1) != 40: return False else: return sha1 home_path = os.path.expanduser("~") nss_path = os.path.join(home_path, ".pki/nssdb") if not os.path.isdir(nss_path): return False if not any( os.path.isfile('%s/certutil' % x) for x in os.environ['PATH'].split(os.pathsep)): xlog.warning( 'please install *libnss3-tools* package to import GoAgent root ca' ) return False sha1 = get_debian_ca_sha1(nss_path) ca_hash = CertUtil.ca_thumbprint.replace(':', '') if sha1 == ca_hash: xlog.info("system cert exist") return # shell command to list all cert # certutil -L -d sql:$HOME/.pki/nssdb # remove old cert first cmd_line = 'certutil -L -d sql:$HOME/.pki/nssdb |grep "GoAgent" && certutil -d sql:$HOME/.pki/nssdb -D -n "%s" ' % ( common_name) os.system(cmd_line) # install new cert cmd_line = 'certutil -d sql:$HOME/.pki/nssdb -A -t "C,," -n "%s" -i "%s"' % ( common_name, ca_file) os.system(cmd_line) return True
def roll_log(self): for i in range(1000): file_name = os.path.join(config.DATA_PATH, "scan_ip.%d.log" % i) if os.path.isfile(file_name): continue xlog.info("scan_ip_log roll %s -> %s", self.log_path, file_name) shutil.move(self.log_path, file_name) return
def remove_channels_for_task(task, channels): xlog.info('remove channels for task %s' % task) for root, dirnames, filenames in os.walk(task + '/cyberecord'): # print root, dirnames, filenames for filename in filenames: path = os.path.join(root, filename) if path.endswith('.record'): xlog.info('remove channels for record %s' % path) remove_channels_for_record(str(path), channels)
def get_connect_interval(): if sys.platform != "win32": return 0 win_version = env_info.win32_version() if win_version == 10: xlog.info("detect Win10, enable connect concurent control, interval:%d", config.connect_interval) return config.connect_interval return 0
def generate_ca_file(): xlog.info("generate CA file:%s", CertUtil.ca_keyfile) key, ca = CertUtil.create_ca() with open(CertUtil.ca_keyfile, 'wb') as fp: fp.write( OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, ca)) fp.write( OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key))
def __init__(self, address, handler, args=()): self.running = True self.server_address = address self.handler = handler self.args = args self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.bind(self.server_address) self.socket.listen(200) xlog.info("server %s:%d started.", address[0], address[1])
def main(): global ready # to profile gae_proxy, run proxy.py, visit some web by proxy, then visit http://127.0.0.1:8084/quit to quit and print result. do_profile = False if do_profile: import cProfile, pstats pr = cProfile.Profile() pr.enable() global __file__ __file__ = os.path.abspath(__file__) if os.path.islink(__file__): __file__ = getattr(os, 'readlink', lambda x: x)(__file__) os.chdir(os.path.dirname(os.path.abspath(__file__))) xlog.basicConfig( level=xlog.DEBUG if config.LISTEN_DEBUGINFO else xlog.INFO, format='%(levelname)s - %(asctime)s %(message)s', datefmt='[%b %d %H:%M:%S]') pre_start() log_info() CertUtil.init_ca() proxy_daemon = LocalProxyServer((config.LISTEN_IP, config.LISTEN_PORT), proxy_handler.GAEProxyHandler) proxy_thread = threading.Thread(target=proxy_daemon.serve_forever) proxy_thread.setDaemon(True) proxy_thread.start() if config.PAC_ENABLE: pac_daemon = LocalProxyServer((config.PAC_IP, config.PAC_PORT), pac_server.PACServerHandler) pac_thread = threading.Thread(target=pac_daemon.serve_forever) pac_thread.setDaemon(True) pac_thread.start() ready = True #checked by launcher.module_init while connect_control.keep_running: time.sleep(1) xlog.info("Exiting gae_proxy module...") proxy_daemon.shutdown() proxy_daemon.server_close() proxy_thread.join() if config.PAC_ENABLE: pac_daemon.shutdown() pac_daemon.server_close() pac_thread.join() ready = False #checked by launcher.module_init xlog.info("Finished Exiting gae_proxy module...") if do_profile: pr.disable() pr.print_stats()
def main(): global ready connect_control.keep_running = True config.load() connect_manager.https_manager.load_config() xlog.debug("## GAEProxy set keep_running: %s", connect_control.keep_running) # to profile gae_proxy, run proxy.py, visit some web by proxy, then visit http://127.0.0.1:8084/quit to quit and print result. do_profile = False if do_profile: import cProfile, pstats pr = cProfile.Profile() pr.enable() global __file__ __file__ = os.path.abspath(__file__) if os.path.islink(__file__): __file__ = getattr(os, 'readlink', lambda x: x)(__file__) os.chdir(os.path.dirname(os.path.abspath(__file__))) xlog.basicConfig(level=xlog.DEBUG if config.LISTEN_DEBUGINFO else xlog.INFO, format='%(levelname)s - %(asctime)s %(message)s', datefmt='[%b %d %H:%M:%S]') pre_start() log_info() CertUtil.init_ca() proxy_daemon = simple_http_server.HTTPServer((config.LISTEN_IP, config.LISTEN_PORT), proxy_handler.GAEProxyHandler) proxy_thread = threading.Thread(target=proxy_daemon.serve_forever) proxy_thread.setDaemon(True) proxy_thread.start() if config.PAC_ENABLE: pac_daemon = simple_http_server.HTTPServer((config.PAC_IP, config.PAC_PORT), pac_server.PACServerHandler) pac_thread = threading.Thread(target=pac_daemon.serve_forever) pac_thread.setDaemon(True) pac_thread.start() ready = True # checked by launcher.module_init while connect_control.keep_running: time.sleep(1) xlog.info("Exiting gae_proxy module...") proxy_daemon.shutdown() proxy_daemon.server_close() proxy_thread.join() if config.PAC_ENABLE: pac_daemon.shutdown() pac_daemon.server_close() pac_thread.join() ready = False # checked by launcher.module_init xlog.debug("## GAEProxy set keep_running: %s", connect_control.keep_running) if do_profile: pr.disable() pr.print_stats()
def req_deploy_handler(self): global deploy_proc req = urlparse.urlparse(self.path).query reqs = urlparse.parse_qs(req, keep_blank_values=True) data = "" log_path = os.path.abspath(os.path.join(current_path, os.pardir, "server", "upload.log")) time_now = datetime.datetime.today().strftime("%H:%M:%S-%a/%d/%b/%Y") if reqs["cmd"] == ["deploy"]: appid = self.postvars["appid"][0] if deploy_proc and deploy_proc.poll() == None: xlog.warn("deploy is running, request denied.") data = '{"res":"deploy is running", "time":"%s"}' % (time_now) else: try: if os.path.isfile(log_path): os.remove(log_path) script_path = os.path.abspath(os.path.join(current_path, os.pardir, "server", "uploader.py")) email = self.postvars["email"][0] passwd = self.postvars["passwd"][0] rc4_passwd = self.postvars["rc4_passwd"][0] deploy_proc = subprocess.Popen([sys.executable, script_path, appid, email, passwd, rc4_passwd]) xlog.info("deploy begin.") data = '{"res":"success", "time":"%s"}' % time_now except Exception as e: data = '{"res":"%s", "time":"%s"}' % (e, time_now) elif reqs["cmd"] == ["cancel"]: if deploy_proc and deploy_proc.poll() == None: deploy_proc.kill() data = '{"res":"deploy is killed", "time":"%s"}' % (time_now) else: data = '{"res":"deploy is not running", "time":"%s"}' % (time_now) elif reqs["cmd"] == ["get_log"]: if deploy_proc and os.path.isfile(log_path): with open(log_path, "r") as f: content = f.read() else: content = "" status = "init" if deploy_proc: if deploy_proc.poll() == None: status = "running" else: status = "finished" data = json.dumps({"status": status, "log": content, "time": time_now}) self.send_response("text/html", data)
def req_deploy_handler(self): global deploy_proc req = urlparse.urlparse(self.path).query reqs = urlparse.parse_qs(req, keep_blank_values=True) data = '' log_path = os.path.abspath(os.path.join(current_path, os.pardir, "server", 'upload.log')) time_now = datetime.datetime.today().strftime('%H:%M:%S-%a/%d/%b/%Y') if reqs['cmd'] == ['deploy']: appid = self.postvars['appid'][0] if deploy_proc and deploy_proc.poll() == None: xlog.warn("deploy is running, request denied.") data = '{"res":"deploy is running", "time":"%s"}' % (time_now) else: try: if os.path.isfile(log_path): os.remove(log_path) script_path = os.path.abspath(os.path.join(current_path, os.pardir, "server", 'uploader.py')) email = self.postvars['email'][0] passwd = self.postvars['passwd'][0] rc4_passwd = self.postvars['rc4_passwd'][0] deploy_proc = subprocess.Popen([sys.executable, script_path, appid, email, passwd, rc4_passwd]) xlog.info("deploy begin.") data = '{"res":"success", "time":"%s"}' % time_now except Exception as e: data = '{"res":"%s", "time":"%s"}' % (e, time_now) elif reqs['cmd'] == ['cancel']: if deploy_proc and deploy_proc.poll() == None: deploy_proc.kill() data = '{"res":"deploy is killed", "time":"%s"}' % (time_now) else: data = '{"res":"deploy is not running", "time":"%s"}' % (time_now) elif reqs['cmd'] == ['get_log']: if deploy_proc and os.path.isfile(log_path): with open(log_path, "r") as f: content = f.read() else: content = "" status = 'init' if deploy_proc: if deploy_proc.poll() == None: status = 'running' else: status = 'finished' data = json.dumps({'status':status,'log':content, 'time':time_now}) self.send_response('text/html', data)
def test_gae(ip_str): xlog.info("==>%s", ip_str) check = Check_frame(ip_str) result = check.check(callback=test_app_head, check_ca=True) if not result: return False check.result.server_type = result return check.result
def test_gws(ip_str): xlog.info("==>%s", ip_str) check = Check_frame(ip_str) result = check.check(callback=test_server_type, check_ca=True) if not result or not "gws" in result: return False check.result.server_type = result return check.result
def check_all_domain(check_ip): with open(os.path.join(current_path, "front_domains.json"), "r") as fd: content = fd.read() cs = json.loads(content) for host in cs: host = "scan1." + host res = check_ip.check_ip(ip, host=host, wait_time=wait_time) if not res or not res.ok: xlog.warn("host:%s fail", host) else: xlog.info("host:%s ok", host)
def check_ssl_cert(ssl_sock): cert = ssl_sock.get_peer_certificate() if not cert: #raise HoneypotError(' certficate is none') raise SSLError("no cert") issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') ssl_cert = cert_util.SSLCert(cert) xlog.info("%s CN:%s", self.ip, ssl_cert.cn) self.result.domain = ssl_cert.cn
def import_debian_ca(common_name, ca_file): def get_debian_ca_sha1(nss_path): commonname = "GoAgent XX-Net - GoAgent" #TODO: here should be GoAgent - XX-Net cmd = ['certutil', '-L','-d', 'sql:%s' % nss_path, '-n', commonname] lines = get_cmd_out(cmd) get_sha1_title = False sha1 = "" for line in lines: if line.endswith("Fingerprint (SHA1):\n"): get_sha1_title = True continue if get_sha1_title: sha1 = line break sha1 = sha1.replace(' ', '').replace(':', '').replace('\n', '') if len(sha1) != 40: return False else: return sha1 home_path = os.path.expanduser("~") nss_path = os.path.join(home_path, ".pki/nssdb") if not os.path.isdir(nss_path): return False if not any(os.path.isfile('%s/certutil' % x) for x in os.environ['PATH'].split(os.pathsep)): xlog.warning('please install *libnss3-tools* package to import GoAgent root ca') return False sha1 = get_debian_ca_sha1(nss_path) ca_hash = CertUtil.ca_thumbprint.replace(':', '') if sha1 == ca_hash: xlog.info("system cert exist") return # shell command to list all cert # certutil -L -d sql:$HOME/.pki/nssdb # remove old cert first cmd_line = 'certutil -L -d sql:$HOME/.pki/nssdb |grep "GoAgent" && certutil -d sql:$HOME/.pki/nssdb -D -n "%s" ' % ( common_name) os.system(cmd_line) # install new cert cmd_line = 'certutil -d sql:$HOME/.pki/nssdb -A -t "C,," -n "%s" -i "%s"' % (common_name, ca_file) os.system(cmd_line) return True
def do_GET(self): xlog.info('PAC from:%s %s %s ', self.address_string(), self.command, self.path) path = urlparse.urlparse(self.path).path # '/proxy.pac' filename = os.path.normpath('./' + path) # proxy.pac if self.path.startswith(('http://', 'https://')): data = b'HTTP/1.1 200\r\nCache-Control: max-age=86400\r\nExpires:Oct, 01 Aug 2100 00:00:00 GMT\r\nConnection: close\r\n' if filename.endswith(('.jpg', '.gif', '.jpeg', '.bmp')): data += b'Content-Type: image/gif\r\n\r\n' + self.onepixel else: data += b'\r\n This is the Pac server, not proxy port, use 8087 as proxy port.' self.wfile.write(data) xlog.info('%s "%s %s HTTP/1.1" 200 -', self.address_string(), self.command, self.path) return # check for '..', which will leak file if re.search(r'(\.{2})', self.path) is not None: self.wfile.write(b'HTTP/1.1 404\r\n\r\n') xlog.warn('%s %s %s haking', self.address_string(), self.command, self.path) return if filename != 'proxy.pac': xlog.warn("pac_server GET %s fail", filename) self.wfile.write(b'HTTP/1.1 404\r\n\r\n') return mimetype = 'text/plain' if self.path.endswith('.pac?flush') or time.time() - os.path.getmtime( get_serving_pacfile()) > config.PAC_EXPIRED: thread.start_new_thread(PacUtil.update_pacfile, (user_pacfile, )) pac_filename = get_serving_pacfile() with open(pac_filename, 'rb') as fp: data = fp.read() host = self.headers.getheader('Host') host, _, port = host.rpartition(":") gae_proxy_proxy = host + ":" + str(config.LISTEN_PORT) pac_proxy = host + ":" + str(config.PAC_PORT) data = data.replace(gae_proxy_listen, gae_proxy_proxy) data = data.replace(pac_listen, pac_proxy) self.wfile.write( ('HTTP/1.1 200\r\nContent-Type: %s\r\nContent-Length: %s\r\n\r\n' % (mimetype, len(data))).encode()) self.wfile.write(data)
def load_range_content(self): if os.path.isfile(user_range_file): self.range_file = user_range_file else: self.range_file = default_range_file xlog.info("load ip range file:%s", self.range_file) fd = open(self.range_file, "r") if not fd: xlog.error("load ip range %s fail", self.range_file) return content = fd.read() fd.close() return content