def pull_log_files(host): start_time = time.time() print 'pull log files from host %s...' % (host.id,) ssh = psi_ssh.SSH( host.ip_address, host.ssh_port, host.stats_ssh_username, host.stats_ssh_password, host.ssh_host_key) dirlist = ssh.list_dir(HOST_LOG_DIR) for filename in dirlist: if re.match(HOST_LOG_FILENAME_PATTERN, filename): try: os.makedirs(os.path.join(LOCAL_LOG_ROOT, host.id)) except OSError: pass ssh.get_file( posixpath.join(HOST_LOG_DIR, filename), os.path.join(LOCAL_LOG_ROOT, host.id, filename)) ssh.close() print 'completed host %s' % (host.id,) return time.time()-start_time
def deploy_geoip_database_autoupdates(host): geo_ip_config_file = 'GeoIP.conf' if os.path.isfile(geo_ip_config_file): print 'deploy geoip database autoupdates to host %s...' % (host.id) ssh = psi_ssh.SSH(host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) ssh.put_file(os.path.join(os.path.abspath('.'), geo_ip_config_file), posixpath.join('/usr/local/etc/', geo_ip_config_file)) # Set up weekly updates cron_filename = '/etc/cron.weekly/update-geoip-db' cron_file_contents = '''#!/bin/sh /usr/local/bin/geoipupdate %s restart''' % (posixpath.join(psi_config.HOST_INIT_DIR, 'psiphonv'), ) ssh.exec_command('echo "%s" > %s' % (cron_file_contents, cron_filename)) ssh.exec_command('chmod +x %s' % (cron_filename, )) # Run the first update ssh.exec_command(cron_filename) ssh.close() host.log('deploy geoip autoupdates')
def change_weekly_crontab_runday(host, weekdaynum): if weekdaynum == None: weekdaynum = datetime.date.isoweekday(datetime.date.today()) if weekdaynum >= 1 or weekdaynum <= 7: cmd = "sed -i 's/^.*weekly.*$/47 6 * * " +str(weekdaynum)+ "\troot\ttest -x \/usr\/sbin\/anacron || ( cd \/ \&\& run-parts --report \/etc\/cron.weekly )/' /etc/crontab" ssh = psi_ssh.SSH( host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) ssh.exec_command(cmd) ssh.close()
def deploy_geoip_database_autoupdates(host): geo_ip_config_file = 'GeoIP.conf' if os.path.isfile(geo_ip_config_file): print 'deploy geoip database autoupdates to host %s...' % (host.id) ssh = psi_ssh.SSH(host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) ssh.put_file(os.path.join(os.path.abspath('.'), geo_ip_config_file), posixpath.join('/usr/local/etc/', geo_ip_config_file)) # Set up weekly updates cron_filename = '/etc/cron.weekly/update-geoip-db' cron_file_contents = None geoipupdate_filename = '/usr/local/bin/geoipupdate' # For TCS, use hot reload and don't restart service if host.is_TCS: cron_file_contents = textwrap.dedent('''#!/bin/sh sleep $((RANDOM %% 10000)) %s %s''' % ( geoipupdate_filename, TCS_PSIPHOND_HOT_RELOAD_SIGNAL_COMMAND, )) else: cron_file_contents = textwrap.dedent('''#!/bin/sh %s %s restart''' % ( geoipupdate_filename, posixpath.join(psi_config.HOST_INIT_DIR, 'psiphonv'), )) ssh.exec_command('echo "%s" > %s' % (cron_file_contents, cron_filename)) ssh.exec_command('chmod +x %s' % (cron_filename, )) # Run the first update ssh.exec_command(geoipupdate_filename) ssh.close() host.log('deploy geoip autoupdates')
def deploy_data(host, host_data, TCS_traffic_rules_set): print 'deploy data to host %s%s...' % ( host.id, " (TCS) " if host.is_TCS else "", ) ssh = psi_ssh.SSH(host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) if host.is_TCS: deploy_TCS_data(ssh, host, host_data, TCS_traffic_rules_set) else: deploy_legacy_data(ssh, host, host_data) ssh.close()
def deploy_routes(host): print 'deploy routes to host %s...' % (host.id, ) ssh = psi_ssh.SSH(host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) ssh.exec_command('mkdir -p %s' % (psi_config.ROUTES_PATH, )) target_filename = posixpath.join( psi_config.ROUTES_PATH, os.path.split(psi_routes.GEO_ROUTES_ARCHIVE_PATH)[1]) ssh.put_file(psi_routes.GEO_ROUTES_ARCHIVE_PATH, target_filename) ssh.exec_command('tar xz -C %s -f %s' % (psi_config.ROUTES_PATH, target_filename)) ssh.close() host.log('deploy routes')
def deploy_build(host, build_filename): print 'deploy %s build to host %s...' % ( build_filename, host.id, ) ssh = psi_ssh.SSH(host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) ssh.exec_command('mkdir -p %s' % (psi_config.UPGRADE_DOWNLOAD_PATH, )) ssh.put_file( build_filename, posixpath.join(psi_config.UPGRADE_DOWNLOAD_PATH, os.path.split(build_filename)[1])) ssh.close()
def do_service_restart(host): sleep(30) if not host.is_TCS: return try: print("restarting 'psiphond.service' on host: %s" % host.id) ssh = psi_ssh.SSH(host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) ssh.exec_command("systemctl restart psiphond.service") except Exception as e: print("Error restarting 'psiphond.service' on host %s: %r" % (host.id, e)) raise host.log("restarted psiphond.service")
def deploy_data(host, host_data, TCS_traffic_rules_set, TCS_OSL_config, TCS_tactics_config_template, TCS_blocklist_csv): print 'deploy data to host %s%s...' % ( host.id, " (TCS) " if host.is_TCS else "", ) ssh = psi_ssh.SSH(host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) if host.is_TCS: deploy_TCS_data(ssh, host, host_data, TCS_traffic_rules_set, TCS_OSL_config, TCS_tactics_config_template, TCS_blocklist_csv) else: deploy_legacy_data(ssh, host, host_data) ssh.close()
def deploy_data(host, host_data): print 'deploy data to host %s...' % (host.id, ) ssh = psi_ssh.SSH(host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) # Stop server, if running, before replacing data file (command may fail) # Disable restarting the server through psi-check-services first ssh.exec_command('touch %s' % (psi_config.HOST_SERVER_STOPPED_LOCK_FILE, )) remote_init_file_path = posixpath.join(psi_config.HOST_INIT_DIR, 'psiphonv') ssh.exec_command('%s stop' % (remote_init_file_path, )) # Copy data file # We upload a compartmentalized version of the master file # containing only the propagation channel IDs and confidential server # information required by each host. file = tempfile.NamedTemporaryFile(delete=False) try: file.write(host_data) file.close() ssh.exec_command('mkdir -p %s' % (posixpath.split(psi_config.DATA_FILE_NAME)[0], )) ssh.put_file(file.name, psi_config.DATA_FILE_NAME) finally: try: os.remove(file.name) except: pass # Restart server after data file updated ssh.exec_command('%s restart' % (remote_init_file_path, )) # Allow psi-check-services to restart the server now that data has been successfully copied # and the server is running again ssh.exec_command('rm %s' % (psi_config.HOST_SERVER_STOPPED_LOCK_FILE, )) ssh.close()
def deploy_implementation(host, servers, discovery_strategy_value_hmac_key, plugins, TCS_psiphond_config_values): print 'deploy implementation to host %s%s...' % ( host.id, " (TCS) " if host.is_TCS else "", ) ssh = psi_ssh.SSH(host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) if host.is_TCS: deploy_TCS_implementation(ssh, host, servers, TCS_psiphond_config_values) else: deploy_legacy_implementation(ssh, host, discovery_strategy_value_hmac_key, plugins) ssh.close()
def install_malware_blacklist(host): psi_ip_blacklist = 'psi_ipblacklist.py' psi_ip_blacklist_host_path = posixpath.join('/usr/local/bin', psi_ip_blacklist) if_up_script_path = '/etc/network/if-up.d/set_blocklist' cron_script_path = '/etc/cron.daily/set_blocklist' ssh = psi_ssh.SSH( host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) ssh.exec_command('apt-get install -y module-assistant xtables-addons-source') ssh.exec_command('module-assistant -i auto-install xtables-addons') ssh.put_file(os.path.join(os.path.abspath('.'), psi_ip_blacklist), psi_ip_blacklist_host_path) ssh.exec_command('chmod +x %s' % (psi_ip_blacklist_host_path,)) ssh.exec_command('ln -s %s %s' % (psi_ip_blacklist_host_path, if_up_script_path)) ssh.exec_command('ln -s %s %s' % (psi_ip_blacklist_host_path, cron_script_path)) ssh.exec_command(psi_ip_blacklist_host_path) ssh.close()
def install_firewall_rules(host, servers, plugins, do_blacklist=True): iptables_rules_path = '/etc/iptables.rules' iptables_rules_contents = ''' *filter -A INPUT -i lo -p tcp -m tcp --dport 7300 -j ACCEPT -A INPUT -i lo -p tcp -m tcp --dport 6379 -j ACCEPT -A INPUT -i lo -p tcp -m tcp --dport 6000 -j ACCEPT''' + ''.join( # tunneled OSSH [''' -A INPUT -i lo -d {0} -p tcp -m state --state NEW -m tcp --dport {1} -j ACCEPT'''.format( str(s.internal_ip_address), str(s.ssh_obfuscated_port)) for s in servers if (s.capabilities['FRONTED-MEEK'] or s.capabilities['UNFRONTED-MEEK']) and s.ssh_obfuscated_port]) + ''.join( # tunneled web requests [''' -A INPUT -i lo -d %s -p tcp -m state --state NEW -m tcp --dport %s -j ACCEPT''' % (str(s.internal_ip_address), str(s.web_server_port)) for s in servers]) + ''' -A INPUT -d 127.0.0.0/8 ! -i lo -j DROP -A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT -A INPUT -p tcp -m state --state NEW -m tcp --dport %s -j ACCEPT''' % (host.ssh_port,) + ''.join( # meek server [''' -A INPUT -d %s -p tcp -m state --state NEW -m tcp --dport %s -m limit --limit 1000/sec -j ACCEPT''' % (str(s.internal_ip_address), str(host.meek_server_port)) for s in servers if (s.capabilities['FRONTED-MEEK'] or s.capabilities['UNFRONTED-MEEK']) and host.meek_server_port]) + ''.join( # web servers [''' -A INPUT -d %s -p tcp -m state --state NEW -m tcp --dport %s -j ACCEPT''' % (str(s.internal_ip_address), str(s.web_server_port)) for s in servers if s.capabilities['handshake']]) + ''.join( # SSH [''' -A INPUT -d {0} -p tcp -m state --state NEW -m tcp --dport {1} -m recent --set --name {2} -A INPUT -d {0} -p tcp -m state --state NEW -m tcp --dport {1} -m recent --update --name {2} --seconds 60 --hitcount 3 -j DROP -A INPUT -d {0} -p tcp -m state --state NEW -m tcp --dport {1} -j ACCEPT'''.format( str(s.internal_ip_address), str(s.ssh_port), 'LIMIT-' + str(s.internal_ip_address).replace('.', '-') + '-' + str(s.ssh_port)) for s in servers if s.capabilities['SSH']]) + ''.join( # OSSH [''' -A INPUT -d {0} -p tcp -m state --state NEW -m tcp --dport {1} -m recent --set --name {2} -A INPUT -d {0} -p tcp -m state --state NEW -m tcp --dport {1} -m recent --update --name {2} --seconds 60 --hitcount 3 -j DROP -A INPUT -d {0} -p tcp -m state --state NEW -m tcp --dport {1} -j ACCEPT'''.format( str(s.internal_ip_address), str(s.ssh_obfuscated_port), 'LIMIT-' + str(s.internal_ip_address).replace('.', '-') + '-' + str(s.ssh_obfuscated_port)) for s in servers if s.capabilities['OSSH']]) + ''.join( # VPN [''' -A INPUT -d {0} -p esp -j ACCEPT -A INPUT -d {0} -p ah -j ACCEPT -A INPUT -d {0} -p udp --dport 500 -j ACCEPT -A INPUT -d {0} -p udp --dport 4500 -j ACCEPT -A INPUT -d {0} -i ipsec+ -p udp -m udp --dport l2tp -j ACCEPT'''.format( str(s.internal_ip_address)) for s in servers if s.capabilities['VPN']]) + ''' -A INPUT -p tcp -j REJECT --reject-with tcp-reset -A INPUT -j DROP -A FORWARD -s 10.0.0.0/8 -p tcp -m multiport --dports 80,443,465,554,587,993,995,1935,5190,7070,8000,8001 -j ACCEPT -A FORWARD -s 10.0.0.0/8 -p udp -m multiport --dports 80,443,465,554,587,993,995,1935,5190,7070,8000,8001 -j ACCEPT -A FORWARD -s 10.0.0.0/8 -p tcp -m multiport --dports 3478,5242,4244,9339 -j ACCEPT -A FORWARD -s 10.0.0.0/8 -p udp -m multiport --dports 3478,5243,7985,9785 -j ACCEPT -A FORWARD -s 10.0.0.0/8 -p tcp -m multiport --dports 110,143,2560,8080,5060,5061,9180,25565 -j ACCEPT -A FORWARD -s 10.0.0.0/8 -p udp -m multiport --dports 110,143,2560,8080,5060,5061,9180,25565 -j ACCEPT -A FORWARD -s 10.0.0.0/8 -d 8.8.8.8 -p tcp --dport 53 -j ACCEPT -A FORWARD -s 10.0.0.0/8 -d 8.8.8.8 -p udp --dport 53 -j ACCEPT -A FORWARD -s 10.0.0.0/8 -d 8.8.4.4 -p tcp --dport 53 -j ACCEPT -A FORWARD -s 10.0.0.0/8 -d 8.8.4.4 -p udp --dport 53 -j ACCEPT -A FORWARD -s 10.0.0.0/8 -d 10.0.0.0/8 -j DROP -A FORWARD -s 10.0.0.0/8 -j REJECT''' + ''.join( # tunneled ossh requests [''' -A OUTPUT -d {0} -o lo -p tcp -m tcp --dport {1} -j ACCEPT -A OUTPUT -s {0} -o lo -p tcp -m tcp --sport {1} -j ACCEPT'''.format( str(s.internal_ip_address), str(s.ssh_obfuscated_port)) for s in servers if s.ssh_obfuscated_port]) + ''.join( # tunneled web requests (always provided, regardless of capabilities) [''' -A OUTPUT -d {0} -o lo -p tcp -m tcp --dport {1} -j ACCEPT -A OUTPUT -s {0} -o lo -p tcp -m tcp --sport {1} -j ACCEPT'''.format( str(s.internal_ip_address), str(s.web_server_port)) for s in servers]) + ''' -A OUTPUT -o lo -p tcp -m tcp --dport 7300 -j ACCEPT -A OUTPUT -o lo -p tcp -m tcp --dport 6379 -m owner --uid-owner root -j ACCEPT -A OUTPUT -o lo -p tcp -m tcp --dport 6000 -m owner --uid-owner root -j ACCEPT -A OUTPUT -o lo -p tcp -m tcp --dport 6379 -m owner --uid-owner www-data -j ACCEPT -A OUTPUT -o lo -p tcp -m tcp --sport 7300 -j ACCEPT -A OUTPUT -o lo -p tcp -m tcp --sport 6379 -j ACCEPT -A OUTPUT -o lo -p tcp -m tcp --sport 6000 -j ACCEPT -A OUTPUT -o lo -j REJECT -A OUTPUT -p tcp -m multiport --dports 53,80,443,465,554,587,993,995,1935,5190,7070,8000,8001 -j ACCEPT -A OUTPUT -p udp -m multiport --dports 53,80,443,465,554,587,993,995,1935,5190,7070,8000,8001 -j ACCEPT -A OUTPUT -p tcp -m multiport --dports 5222,5223,5224,5228,5229,5230,5269,14259 -j ACCEPT -A OUTPUT -p udp -m multiport --dports 5222,5223,5224,5228,5229,5230,5269,14259 -j ACCEPT -A OUTPUT -p tcp -m multiport --dports 3478,5242,4244,9339 -j ACCEPT -A OUTPUT -p udp -m multiport --dports 3478,5243,7985,9785 -j ACCEPT -A OUTPUT -p tcp -m multiport --dports 110,143,2560,8080,5060,5061,9180,25565 -j ACCEPT -A OUTPUT -p udp -m multiport --dports 110,143,2560,8080,5060,5061,9180,25565 -j ACCEPT -A OUTPUT -p udp -m udp --dport 123 -j ACCEPT -A OUTPUT -p tcp -m tcp --sport %s -j ACCEPT''' % (host.ssh_port,) + ''.join( # tunneled ossh requests on NATed servers [''' -A OUTPUT -d %s -p tcp -m tcp --dport %s -j ACCEPT''' % (str(s.internal_ip_address), str(s.ssh_obfuscated_port)) for s in servers if s.ssh_obfuscated_port and (s.ip_address != s.internal_ip_address)]) + ''.join( # tunneled web requests on NATed servers don't go out lo [''' -A OUTPUT -d %s -p tcp -m tcp --dport %s -j ACCEPT''' % (str(s.internal_ip_address), str(s.web_server_port)) for s in servers if s.ip_address != s.internal_ip_address]) + ''.join( # meek server [''' -A OUTPUT -s %s -p tcp -m tcp --sport %s -j ACCEPT''' % (str(s.internal_ip_address), str(host.meek_server_port)) for s in servers if host.meek_server_port]) + ''.join( # web servers [''' -A OUTPUT -s %s -p tcp -m tcp --sport %s -j ACCEPT''' % (str(s.internal_ip_address), str(s.web_server_port)) for s in servers if s.web_server_port]) + ''.join( # SSH [''' -A OUTPUT -s %s -p tcp -m tcp --sport %s -j ACCEPT''' % (str(s.internal_ip_address), str(s.ssh_port)) for s in servers if s.ssh_port]) + ''.join( # OSSH [''' -A OUTPUT -s %s -p tcp -m tcp --sport %s -j ACCEPT''' % (str(s.internal_ip_address), str(s.ssh_obfuscated_port)) for s in servers if s.ssh_obfuscated_port]) + ''.join( # VPN [''' -A OUTPUT -s {0} -p esp -j ACCEPT -A OUTPUT -s {0} -p ah -j ACCEPT -A OUTPUT -s {0} -p udp --sport 500 -j ACCEPT -A OUTPUT -s {0} -p udp --sport 4500 -j ACCEPT -A OUTPUT -s {0} -o ipsec+ -p udp -m udp --dport l2tp -j ACCEPT'''.format( str(s.internal_ip_address)) for s in servers if s.capabilities['VPN']]) + ''.join( [''' -A OUTPUT -s %s -p tcp -m tcp --tcp-flags ALL ACK,RST -j ACCEPT''' % (str(s.internal_ip_address), ) for s in servers]) + ''' -A OUTPUT -j REJECT COMMIT *nat''' + ''.join( # Port forward from 443 to web servers # NOTE: exclude for servers with meek capability (or is fronted) and meek_server_port is 443 # or OSSH is running on 443 [''' -A PREROUTING -i eth+ -p tcp -d %s --dport 443 -j DNAT --to-destination :%s''' % (str(s.internal_ip_address), str(s.web_server_port)) for s in servers if s.capabilities['handshake'] and not ( ((s.capabilities['FRONTED-MEEK'] or s.capabilities['UNFRONTED-MEEK']) and int(host.meek_server_port) == 443) or (s.capabilities['OSSH'] and int(s.ssh_obfuscated_port) == 443))]) + ''.join( # Port forward alternate ports [''' -A PREROUTING -i eth+ -p tcp -d %s --dport %s -j DNAT --to-destination :%s''' % (str(s.internal_ip_address), str(alternate), str(s.ssh_obfuscated_port)) for s in servers if s.alternate_ssh_obfuscated_ports for alternate in s.alternate_ssh_obfuscated_ports]) + ''' -A POSTROUTING -s 10.0.0.0/8 -o eth+ -j MASQUERADE''' + ''.join( # tunneled web requests on NATed servers need to be redirected to the servers' internal ip addresses [''' -A OUTPUT -p tcp -m tcp -d %s --dport %s -j DNAT --to-destination %s''' % (str(s.ip_address), str(s.web_server_port), str(s.internal_ip_address)) for s in servers if s.ip_address != s.internal_ip_address]) + ''' COMMIT ''' for plugin in plugins: if hasattr(plugin, 'iptables_rules_contents'): iptables_rules_contents = plugin.iptables_rules_contents(host, servers) # NOTE that we restart fail2ban after applying firewall rules because iptables-restore # flushes iptables which will remove any chains and rules that fail2ban creates on starting up if_up_script_path = '/etc/network/if-up.d/firewall' if_up_script_contents = '''#!/bin/sh iptables-restore < %s /etc/init.d/fail2ban restart ''' % (iptables_rules_path,) ssh_ports = set([str(host.ssh_port)]) for server in servers: ssh_ports.add(str(server.ssh_port)) if server.capabilities['SSH'] else None ssh_ports.add(str(server.ssh_obfuscated_port)) if server.capabilities['OSSH'] else None fail2ban_local_path = '/etc/fail2ban/jail.local' fail2ban_local_contents = textwrap.dedent(''' [ssh] port = {0} [ssh-ddos] port = {0} '''.format(','.join(ssh_ports))) meek_server_egress_ips = set([(str(s.egress_ip_address)) for s in servers if (s.capabilities['FRONTED-MEEK'] or s.capabilities['UNFRONTED-MEEK'])]) if meek_server_egress_ips: fail2ban_local_contents = textwrap.dedent(''' [DEFAULT] ignoreip = 127.0.0.1/8 {0} '''.format(' '.join(meek_server_egress_ips))) + fail2ban_local_contents ssh = psi_ssh.SSH( host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) ssh.exec_command('echo "%s" > %s' % (iptables_rules_contents, iptables_rules_path)) ssh.exec_command('echo "%s" > %s' % (if_up_script_contents, if_up_script_path)) ssh.exec_command('chmod +x %s' % (if_up_script_path,)) ssh.exec_command('echo "%s" > %s' % (fail2ban_local_contents, fail2ban_local_path)) ssh.exec_command(if_up_script_path) ssh.close() if do_blacklist: install_malware_blacklist(host)
def install_psi_limit_load(host, servers): # NOTE: only disabling SSH/OSSH/IKE since disabling the web server from external access # would also prevent current VPN users from accessing the web server. rules = ( # SSH [' INPUT -d %s -p tcp -m state --state NEW -m tcp --dport %s -j REJECT --reject-with tcp-reset' % (str(s.internal_ip_address), str(s.ssh_port)) for s in servers if s.capabilities['SSH']] + # OSSH # NOTE: that this also disables new tunneled OSSH connections ie through meek [' INPUT -d %s -p tcp -m state --state NEW -m tcp --dport %s -j REJECT --reject-with tcp-reset' % (str(s.internal_ip_address), str(s.ssh_obfuscated_port)) for s in servers if s.ssh_obfuscated_port] + # VPN [' INPUT -d %s -p udp --dport 500 -j DROP' % (str(s.internal_ip_address), ) for s in servers if s.capabilities['VPN']] ) disable_services = '\n '.join(['iptables -I' + rule for rule in rules]) enable_services = '\n '.join(['iptables -D' + rule for rule in rules]) script = ''' #!/bin/bash threshold_load_per_cpu=4 threshold_mem=20 threshold_swap=20 while true; do loaded_cpu=0 num_cpu=`grep 'model name' /proc/cpuinfo | wc -l` threshold_cpu=$(($threshold_load_per_cpu * $num_cpu - 1)) load_cpu=`uptime | cut -d , -f 4 | cut -d : -f 2 | awk -F \. '{print $1}'` if [ "$load_cpu" -ge "$threshold_cpu" ]; then loaded_cpu=1 logger psi_limit_load: CPU load threshold reached. break fi free=$(free | grep "buffers/cache" | awk '{print $4/($3+$4) * 100.0}') loaded_mem=$(echo "$free<$threshold_mem" | bc) if [ $loaded_mem -eq 1 ]; then logger psi_limit_load: Free memory load threshold reached. fi loaded_swap=0 total_swap=$(free | grep "Swap" | awk '{print $2}') if [ $total_swap -ne 0 ]; then free_swap=$(free | grep "Swap" | awk '{print $4/$2 * 100.0}') loaded_swap=$(echo "$free_swap<$threshold_swap" | bc) if [ $loaded_swap -eq 1]; then logger psi_limit_load: Swap threshold reached. fi fi break done if [ $loaded_cpu -eq 1 ] || [ $loaded_mem -eq 1 ] || [ $loaded_swap -eq 1 ]; then %s %s service xinetd stop else if [[ -z $(pgrep xinetd) ]]; then service xinetd restart fi %s fi exit 0 ''' % (enable_services, disable_services, enable_services) ssh = psi_ssh.SSH( host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) ssh.exec_command('apt-get install -y bc') psi_limit_load_host_path = '/usr/local/sbin/psi_limit_load' file = tempfile.NamedTemporaryFile(delete=False) file.write(script) file.close() ssh.put_file(file.name, psi_limit_load_host_path) os.remove(file.name) ssh.exec_command('chmod +x %s' % (psi_limit_load_host_path,)) cron_file = '/etc/cron.d/psi-limit-load' ssh.exec_command('echo "SHELL=/bin/sh" > %s;' % (cron_file,) + 'echo "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" >> %s;' % (cron_file,) + 'echo "* * * * * root %s" >> %s' % (psi_limit_load_host_path, cron_file))
def deploy_implementation(host, discovery_strategy_value_hmac_key, plugins): print 'deploy implementation to host %s...' % (host.id, ) ssh = psi_ssh.SSH(host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) # Copy server source code for (dir, filenames) in SOURCE_FILES: ssh.exec_command('mkdir -p %s' % (posixpath.join(psi_config.HOST_SOURCE_ROOT, *dir), )) for filename in filenames: ssh.put_file( os.path.join(os.path.abspath('..'), *(dir + (filename, ))), posixpath.join(psi_config.HOST_SOURCE_ROOT, *(dir + (filename, )))) ssh.exec_command( 'rm %s' % (posixpath.join(psi_config.HOST_SOURCE_ROOT, *(dir + ('*.pyc', ))), )) ssh.exec_command('chmod +x %s' % (posixpath.join( psi_config.HOST_SOURCE_ROOT, 'Server', 'psi_web.py'), )) ssh.exec_command('chmod +x %s' % (posixpath.join( psi_config.HOST_SOURCE_ROOT, 'Server', 'psi_auth.py'), )) ssh.exec_command('chmod +x %s' % (posixpath.join( psi_config.HOST_SOURCE_ROOT, 'Server', 'psi-check-services'), )) remote_ip_down_file_path = posixpath.join(psi_config.HOST_IP_DOWN_DIR, 'psi-ip-down') ssh.put_file(os.path.join(os.path.abspath('..'), 'Server', 'psi-ip-down'), remote_ip_down_file_path) ssh.exec_command('chmod +x %s' % (remote_ip_down_file_path, )) remote_init_file_path = posixpath.join(psi_config.HOST_INIT_DIR, 'psiphonv') ssh.put_file(os.path.join(os.path.abspath('..'), 'Server', 'psi-init'), remote_init_file_path) ssh.exec_command('chmod +x %s' % (remote_init_file_path, )) ssh.exec_command('update-rc.d %s defaults' % ('psiphonv', )) # Patch PAM config to use psi_auth.py ssh.exec_command( 'grep psi_auth.py /etc/pam.d/sshd || sed -i \'s/@include common-auth/auth sufficient pam_exec.so expose_authtok seteuid quiet \\/opt\\/PsiphonV\\/Server\\/psi_auth.py\\n@include common-auth/\' /etc/pam.d/sshd' ) # Restart server after source code updated ssh.exec_command('%s restart' % (remote_init_file_path, )) # Set up meek-server if enabled for this host if host.meek_server_port: ssh.exec_command( 'mkdir -p /opt/gocode/src/bitbucket.org/psiphon/psiphon-circumvention-system/' ) ssh.exec_command( 'ln -s %s /opt/gocode/src/bitbucket.org/psiphon/psiphon-circumvention-system/' % (posixpath.join(psi_config.HOST_SOURCE_ROOT, 'go'), )) ssh.exec_command( 'cd %s && GOBIN=. GOPATH=/opt/gocode/ go get' % (posixpath.join( psi_config.HOST_SOURCE_ROOT, 'go', 'meek-server'), )) meek_remote_init_file_path = posixpath.join(psi_config.HOST_INIT_DIR, 'meek-server') ssh.put_file( os.path.join(os.path.abspath('..'), 'go', 'meek-server', 'meek-server-init'), meek_remote_init_file_path) ssh.exec_command('chmod +x %s' % (meek_remote_init_file_path, )) ssh.exec_command('update-rc.d %s defaults' % ('meek-server', )) ssh.exec_command('echo \'%s\' > /etc/meek-server.json' % (json.dumps({ 'Port': int(host.meek_server_port), 'ListenTLS': True if host.meek_server_fronting_domain else False, 'CookiePrivateKeyBase64': host.meek_cookie_encryption_private_key, 'ObfuscatedKeyword': host.meek_server_obfuscated_key, 'GeoIpServicePort': psi_config.GEOIP_SERVICE_PORT, 'ClientIpAddressStrategyValueHmacKey': discovery_strategy_value_hmac_key }), )) ssh.exec_command('%s restart' % (meek_remote_init_file_path, )) # Install the cron job that calls psi-check-services cron_file = '/etc/cron.d/psi-check-services' ssh.exec_command( 'echo "SHELL=/bin/sh" > %s;' % (cron_file, ) + 'echo "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" >> %s;' % (cron_file, ) + 'echo "*/5 * * * * root %s" >> %s' % (posixpath.join(psi_config.HOST_SOURCE_ROOT, 'Server', 'psi-check-services'), cron_file)) # Copy the rate-limiting scripts remote_rate_limit_start_file_path = posixpath.join( psi_config.HOST_IP_UP_DIR, 'rate-limit') ssh.put_file( os.path.join(os.path.abspath('..'), 'Server', 'rate-limit-start'), remote_rate_limit_start_file_path) ssh.exec_command('chmod +x %s' % (remote_rate_limit_start_file_path, )) remote_rate_limit_end_file_path = posixpath.join( psi_config.HOST_IP_DOWN_DIR, 'rate-limit') ssh.put_file( os.path.join(os.path.abspath('..'), 'Server', 'rate-limit-end'), remote_rate_limit_end_file_path) ssh.exec_command('chmod +x %s' % (remote_rate_limit_end_file_path, )) for plugin in plugins: if hasattr(plugin, 'deploy_implementation'): plugin.deploy_implementation(ssh) ssh.close()
def install_user_count_and_log(host, servers): server_details = {} for server in servers: server_details[server.id] = {"commands": {}, "fronted": server.capabilities["FRONTED-MEEK"]} server_details[server.id]["commands"]["obfuscated_ssh_users_command"] = "netstat -tpn | grep \"%s:%d \" | grep sshd | grep ESTABLISHED | wc -l" % (server.ip_address, int(server.ssh_obfuscated_port)) server_details[server.id]["commands"]["meek_users_command"] = "netstat -tpn | grep \"%s:%d *%s\" | grep sshd | grep ESTABLISHED | wc -l" % (server.ip_address, int(server.ssh_obfuscated_port), server.ip_address) server_details[server.id]["commands"]["ssh_users_command"] = "" if not server.capabilities["SSH"] else \ "netstat -tpn | grep \"%s:%d \" | grep sshd | grep ESTABLISHED | wc -l" % (server.ip_address, int(server.ssh_port)) vpn_users_command = "ifconfig | grep ppp | wc -l" script = ''' #!/usr/bin/env python import json from datetime import datetime import os import syslog import time import random from collections import defaultdict time.sleep(random.choice(range(0,50))) log_record = { "event_name": "user_count", "timestamp": datetime.utcnow().isoformat() + "Z", "host_id": "%s", "region": "%s", "provider": "%s", "datacenter": "%s", "users": { "obfuscated_ssh": { "servers": defaultdict(dict), "total": 0 }, "ssh": { "servers": defaultdict(dict), "total": 0 }, "vpn": 0, "total": 0 } } server_details = %s vpn_users_command = "%s" for server_id in server_details: log_record["users"]["obfuscated_ssh"]["servers"][server_id]["total"] = int(os.popen(server_details[server_id]["commands"]["obfuscated_ssh_users_command"]).read().strip()) log_record["users"]["obfuscated_ssh"]["total"] += log_record["users"]["obfuscated_ssh"]["servers"][server_id]["total"] log_record["users"]["obfuscated_ssh"]["servers"][server_id]["meek"] = int(os.popen(server_details[server_id]["commands"]["meek_users_command"]).read().strip()) log_record["users"]["obfuscated_ssh"]["servers"][server_id]["direct"] = max(0, log_record["users"]["obfuscated_ssh"]["servers"][server_id]["total"] - log_record["users"]["obfuscated_ssh"]["servers"][server_id]["meek"]) log_record["users"]["obfuscated_ssh"]["servers"][server_id]["fronted"] = server_details[server_id]["fronted"] ssh_users_command = server_details[server_id]["commands"]["ssh_users_command"] log_record["users"]["ssh"]["servers"][server_id]["total"] = 0 if len(ssh_users_command) == 0 else int(os.popen(ssh_users_command).read().strip()) log_record["users"]["ssh"]["total"] += log_record["users"]["ssh"]["servers"][server_id]["total"] log_record["users"]["vpn"] = int(os.popen(vpn_users_command).read().strip()) log_record["users"]["total"] = log_record["users"]["obfuscated_ssh"]["total"] + log_record["users"]["ssh"]["total"] + log_record["users"]["vpn"] syslog.openlog('psiphon-user-count') syslog.syslog(syslog.LOG_INFO, json.dumps(log_record)) ''' % (host.id, host.region, host.provider, host.datacenter_name, server_details, vpn_users_command) ssh = psi_ssh.SSH(host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) psi_count_users_host_path = '/usr/local/sbin/psi_count_users' file = tempfile.NamedTemporaryFile(delete=False) file.write(script) file.close() ssh.put_file(file.name, psi_count_users_host_path) os.remove(file.name) ssh.exec_command('chmod +x %s' % (psi_count_users_host_path,)) cron_file = '/etc/cron.d/psi-count-users' ssh.exec_command('echo "SHELL=/bin/sh" > %s;' % (cron_file,) + 'echo "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" >> %s;' % (cron_file,) + 'echo "* * * * * root python %s" >> %s' % (psi_count_users_host_path, cron_file))
def install_host(host, servers, existing_server_ids, plugins): install_firewall_rules(host, servers, plugins) install_psi_limit_load(host, servers) install_user_count_and_log(host, servers) # NOTE: # For partially configured hosts we need to completely reconfigure # all files because we use a counter in the xl2tpd config files that is not # directly tied to the IP Address in the database. ssh = psi_ssh.SSH( host.ip_address, host.ssh_port, host.ssh_username, host.ssh_password, host.ssh_host_key) # # Generate and upload ipsec config files # file = tempfile.NamedTemporaryFile(delete=False) file.write(make_ipsec_config_file_base_contents(len(servers))) file.close() ssh.put_file(file.name, '/etc/ipsec.conf') os.remove(file.name) for index, server in enumerate(servers): ssh.exec_command( make_ipsec_config_file_connection_command(index, server.internal_ip_address)) ssh.exec_command(make_ipsec_secrets_file_command()) # # Generate and upload xl2tpd config files and init script # # Stop the default instance first ssh.exec_command('/etc/init.d/xl2tpd stop') for index, server in enumerate(servers): ssh.exec_command( make_xl2tpd_config_file_command(index, server.internal_ip_address)) ssh.exec_command(make_xl2tpd_options_file_command()) ssh.exec_command(make_xl2tpd_chap_secrets_file_command()) file = tempfile.NamedTemporaryFile(delete=False) file.write(make_xl2tpd_initd_file_contents(len(servers))) file.close() ssh.put_file(file.name, '/etc/init.d/xl2tpd') os.remove(file.name) # # Restart the IPSec and xl2tpd services # ssh.exec_command('/etc/init.d/ipsec restart') ssh.exec_command('/etc/init.d/xl2tpd restart') # # Upload and install patched Open SSH # ssh.exec_command('rm -rf %(key)s; mkdir -p %(key)s' % {"key": psi_config.HOST_OSSH_SRC_DIR}) remote_ossh_file_path = posixpath.join(psi_config.HOST_OSSH_SRC_DIR, 'ossh.tar.gz') ssh.put_file(os.path.join(os.path.abspath('..'), 'Server', '3rdParty', 'ossh.tar.gz'), remote_ossh_file_path) ssh.exec_command('cd %s; tar xfz ossh.tar.gz; ./configure --with-pam > /dev/null; make > /dev/null && make install > /dev/null' %(psi_config.HOST_OSSH_SRC_DIR,)) # # Upload and install badvpn-udpgw # ssh.exec_command('apt-get install -y cmake') ssh.exec_command('rm -rf %(key)s; mkdir -p %(key)s' % {"key": psi_config.HOST_BADVPN_SRC_DIR}) remote_badvpn_file_path = posixpath.join(psi_config.HOST_BADVPN_SRC_DIR, 'badvpn.tar.gz') ssh.put_file(os.path.join(os.path.abspath('..'), 'Server', '3rdParty', 'badvpn.tar.gz'), remote_badvpn_file_path) ssh.exec_command('cd %s; tar xfz badvpn.tar.gz; mkdir build; cd build; cmake ../badvpn -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_NOTHING_BY_DEFAULT=1 -DBUILD_UDPGW=1 > /dev/null; make > /dev/null && make install > /dev/null' %(psi_config.HOST_BADVPN_SRC_DIR,)) remote_init_file_path = posixpath.join(psi_config.HOST_INIT_DIR, 'badvpn-udpgw') ssh.put_file(os.path.join(os.path.abspath('..'), 'Server', 'udpgw-init'), remote_init_file_path) ssh.exec_command('chmod +x %s' % (remote_init_file_path,)) ssh.exec_command('update-rc.d %s defaults' % ('badvpn-udpgw',)) ssh.exec_command('%s restart' % (remote_init_file_path,)) # # Generate and upload sshd_config files and xinetd.conf # for server in servers: # Generate SSH credentials and SSH host key here because we need to create them # on the server and use them in the sshd_config files. # They will be updated in the database below. if (server.ssh_username is None or server.ssh_password is None): server.ssh_username = '******' % ( binascii.hexlify(os.urandom(SSH_RANDOM_USERNAME_SUFFIX_BYTE_LENGTH)),) server.ssh_password = binascii.hexlify(os.urandom(SSH_PASSWORD_BYTE_LENGTH)) if server.ssh_host_key is None: ssh.exec_command('rm /etc/ssh/ssh_host_rsa_key.psiphon_ssh_%s' % (server.internal_ip_address,)) ssh.exec_command('ssh-keygen -t rsa -N \"\" -f /etc/ssh/ssh_host_rsa_key.psiphon_ssh_%s' % (server.internal_ip_address,)) try: # TODO: use temp dir? ssh.get_file('/etc/ssh/ssh_host_rsa_key.psiphon_ssh_%s.pub' % (server.internal_ip_address,), 'ssh_host_key') with open('ssh_host_key') as file: key = file.read() finally: os.remove('ssh_host_key') # expected format: ssh-rsa <base64> username@host # output format: ssh-rsa <base64> server.ssh_host_key = ' '.join(key.split(' ')[:2]) # NOTE unconditionally attempt to create the user. It's OK if it fails because # the user already exists ssh.exec_command('useradd -d /dev/null -s /bin/false %s && echo \"%s:%s\"|chpasswd' % ( server.ssh_username, server.ssh_username, server.ssh_password)) ssh.exec_command(make_sshd_config_file_command(server.internal_ip_address, server.ssh_username)) if server.ssh_obfuscated_port is not None: if server.ssh_obfuscated_key is None: server.ssh_obfuscated_key = binascii.hexlify(os.urandom(SSH_OBFUSCATED_KEY_BYTE_LENGTH)) ssh.exec_command(make_obfuscated_sshd_config_file_command(server.internal_ip_address, server.ssh_username, server.ssh_obfuscated_port, server.ssh_obfuscated_key)) # NOTE we do not write the ssh host key back to the server because it is generated # on the server in the first place. ssh.exec_command(make_xinetd_config_file_command(servers)) # # Restart the xinetd service # ssh.exec_command('/etc/init.d/xinetd restart') # # Restart some services regularly # cron_file = '/etc/cron.d/psi-restart-services' ssh.exec_command('echo "SHELL=/bin/sh" > %s;' % (cron_file,) + 'echo "PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin" >> %s;' % (cron_file,) + 'echo "1 * * * * root killall -9 %s && %s restart" >> %s;' % ('xinetd', '/etc/init.d/xinetd', cron_file) + 'echo "2 * * * * root killall -9 %s && %s restart" >> %s' % ('badvpn-udpgw', '/etc/init.d/badvpn-udpgw', cron_file)) # # Add required packages and Python modules # ssh.exec_command('apt-get install -y python-pip libffi-dev') ssh.exec_command('pip install pyOpenSSL') ssh.exec_command('pip install hiredis') ssh.exec_command('pip install redis') ssh.exec_command('pip install iso8601') ssh.exec_command('pip install --upgrade cffi') ssh.exec_command('apt-get install -y redis-server mercurial git') install_geoip_database(ssh) for plugin in plugins: if hasattr(plugin, 'install_host'): plugin.install_host(ssh) ssh.close() # # Generate unique server alias and web server credentials # for server in servers: if server.id is None: server.id = generate_unique_server_id(existing_server_ids) if server.web_server_secret is None: server.web_server_secret = generate_web_server_secret() # Generated output is PEM, e.g., # # '-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAvAmIPX5kzPz... # EZ3bCbVPQNP6ZnC6EONGuGTDgTTU30\n-----END RSA PRIVATE KEY-----\n' # # We strip the BEGIN/END lines and remove newlines in the database # format. if (server.web_server_certificate is None or server.web_server_private_key is None): cert_pem, key_pem = generate_self_signed_certificate() server.web_server_private_key = ''.join(key_pem.split('\n')[1:-2]) server.web_server_certificate = ''.join(cert_pem.split('\n')[1:-2])