def _disconnected(self, client): org_id = client['org_id'] user_id = client['user_id'] remote_ip = client['real_address'] org = self.get_org(org_id) if org: user = org.get_user(user_id, fields=('_id', 'name')) if user: user.audit_event( 'user_connection', 'User disconnected from "%s"' % self.server.name, remote_addr=remote_ip, ) monitoring.insert_point('user_disconnections', { 'host': settings.local.host.name, 'server': self.server.name, }, { 'user': user.name, 'remote_ip': remote_ip, }) if self.route_clients: messenger.publish('client', { 'state': False, 'server_id': self.server.id, 'virt_address': client['virt_address'], 'virt_address6': client['virt_address6'], 'host_address': settings.local.host.local_addr, 'host_address6': settings.local.host.local_addr6, }) self.instance_com.push_output( 'User disconnected user_id=%s' % client['user_id']) self.send_event()
def after_request(response): if not flask.g.authed: raise ValueError('Request not authorized') resp_time = int((time.time() - flask.g.start) * 1000) db_time = int(flask.g.query_time * 1000) db_reads = flask.g.query_count db_writes = flask.g.write_count response.headers.add('Execution-Time', resp_time) response.headers.add('Query-Time', db_time) response.headers.add('Query-Count', db_reads) response.headers.add('Write-Count', db_writes) if not flask.request.path.startswith('/event'): monitoring.insert_point('requests', { 'host': settings.local.host.name, }, { 'path': flask.request.path, 'remote_ip': utils.get_remote_addr(), 'response_time': resp_time, 'db_time': db_time, 'db_reads': db_reads, 'db_writes': db_writes, }) return response
def after_request(response): if settings.app.check_requests and not flask.g.valid: raise ValueError("Request not authorized") resp_time = int((time.time() - flask.g.start) * 1000) db_time = int(flask.g.query_time * 1000) db_reads = flask.g.query_count db_writes = flask.g.write_count response.headers.add("Execution-Time", resp_time) response.headers.add("Query-Time", db_time) response.headers.add("Query-Count", db_reads) response.headers.add("Write-Count", db_writes) response.headers.add("X-Frame-Options", "DENY") if not flask.request.path.startswith("/event"): monitoring.insert_point( "requests", {"host": settings.local.host.name}, { "path": flask.request.path, "remote_ip": utils.get_remote_addr(), "response_time": resp_time, "db_time": db_time, "db_reads": db_reads, "db_writes": db_writes, }, ) return response
def _watch_thread(self): try: while True: self.cur_timestamp = utils.now() timestamp_ttl = self.cur_timestamp - datetime.timedelta( seconds=180) for client_id, (timestamp, _, _) in self.client_bytes.items(): if timestamp < timestamp_ttl: self.client_bytes.pop(client_id, None) self.bytes_lock.acquire() bytes_recv = self.bytes_recv bytes_sent = self.bytes_sent self.bytes_recv = 0 self.bytes_sent = 0 self.bytes_lock.release() monitoring.insert_point('server_bandwidth', { 'host': settings.local.host.name, 'server': self.server.name, }, { 'bytes_sent': bytes_sent, 'bytes_recv': bytes_recv, }) monitoring.insert_point('server', { 'host': settings.local.host.name, 'server': self.server.name, }, { 'device_count': self.clients.clients.count({}), }) if bytes_recv != 0 or bytes_sent != 0: self.server.bandwidth.add_data( utils.now(), bytes_recv, bytes_sent) yield interrupter_sleep(self.bandwidth_rate) if self.instance.sock_interrupt: return except GeneratorExit: raise except: try: self.push_output('ERROR Management rate thread error') except: pass logger.exception('Error in management rate thread', 'server', server_id=self.server.id, instance_id=self.instance.id, ) self.instance.stop_process()
def task(self): if settings.app.demo_mode: return try: cursor = self.hosts_collection.find({}, { '_id': True, 'status': True, 'ping_timestamp': True, 'server_count': True, 'device_count': True, }) yield now = utils.now() ttl = datetime.timedelta(seconds=settings.app.host_ping_ttl) ttl_timestamp = {'$lt': now - ttl} server_count = 0 device_count = 0 for doc in cursor: if doc.get('status') == ONLINE: server_count += doc.get('server_count') or 0 device_count += doc.get('device_count') or 0 if doc.get('ping_timestamp') and \ now - doc['ping_timestamp'] > ttl: response = self.hosts_collection.update({ '_id': doc['_id'], 'ping_timestamp': ttl_timestamp, }, {'$set': { 'status': OFFLINE, 'ping_timestamp': None, }}) yield if response['updatedExisting']: event.Event(type=HOSTS_UPDATED) yield monitoring.insert_point('cluster', {}, { 'server_count': server_count, 'device_count': device_count, }) except GeneratorExit: raise except: logger.exception('Error checking host status', 'runners')
def _disconnected(self, client): org_id = client['org_id'] user_id = client['user_id'] remote_ip = client['real_address'] org = self.get_org(org_id) if org: user = org.get_user(user_id, fields=('_id', 'name')) if user: user.audit_event( 'user_connection', 'User disconnected from "%s"' % self.server.name, remote_addr=remote_ip, ) monitoring.insert_point('user_disconnections', { 'host': settings.local.host.name, 'server': self.server.name, }, { 'user': user.name, 'remote_ip': remote_ip, }) plugins.event( 'user_disconnected', host_id=settings.local.host_id, server_id=self.server.id, org_id=org.id, user_id=user.id, host_name=settings.local.host.name, server_name=self.server.name, org_name=org.name, user_name=user.name, remote_ip=remote_ip, ) # if self.server.route_clients and not client.get('ignore_routes'): # messenger.publish('client', { # 'state': False, # 'server_id': self.server.id, # 'virt_address': client['virt_address'], # 'virt_address6': client['virt_address6'], # 'host_address': self.route_addr, # 'host_address6': settings.local.host.local_addr6, # }) self.instance_com.push_output( 'User disconnected user_id=%s' % client['user_id']) self.send_event()
def after_request(response): if settings.app.check_requests and not flask.g.valid: raise ValueError('Request not authorized') response.headers.add('X-Frame-Options', 'DENY') if settings.app.server_ssl: response.headers.add('Strict-Transport-Security', 'max-age=31536000') if not flask.request.path.startswith('/event'): monitoring.insert_point('requests', { 'host': settings.local.host.name, }, { 'path': flask.request.path, 'remote_ip': utils.get_remote_addr(), 'response_time': int((time.time() - flask.g.start) * 1000), }) return response
def after_request(response): if settings.app.check_requests and not flask.g.valid: raise ValueError('Request not authorized') response.headers.add('X-Frame-Options', 'DENY') if settings.app.server_ssl or settings.app.reverse_proxy: response.headers.add('Strict-Transport-Security', 'max-age=31536000') if not flask.request.path.startswith('/event'): monitoring.insert_point( 'requests', { 'host': settings.local.host.name, }, { 'path': flask.request.path, 'remote_ip': utils.get_remote_addr(), 'response_time': int((time.time() - flask.g.start) * 1000), }) return response
def _watch_thread(self): try: while True: self.cur_timestamp = utils.now() timestamp_ttl = self.cur_timestamp - datetime.timedelta(seconds=180) for client_id, (timestamp, _, _) in self.client_bytes.items(): if timestamp < timestamp_ttl: self.client_bytes.pop(client_id, None) self.bytes_lock.acquire() bytes_recv = self.bytes_recv bytes_sent = self.bytes_sent self.bytes_recv = 0 self.bytes_sent = 0 self.bytes_lock.release() monitoring.insert_point( "server_bandwidth", {"host": settings.local.host.name, "server": self.server.name}, {"bytes_sent": bytes_sent, "bytes_recv": bytes_recv}, ) if bytes_recv != 0 or bytes_sent != 0: self.server.bandwidth.add_data(utils.now(), bytes_recv, bytes_sent) yield interrupter_sleep(self.bandwidth_rate) if self.instance.sock_interrupt: return except GeneratorExit: raise except: self.push_output("ERROR Management rate thread error") logger.exception( "Error in management rate thread", "server", server_id=self.server.id, instance_id=self.instance.id ) self.instance.stop_process()
def after_request(response): resp_time = int((time.time() - flask.g.start) * 1000) db_time = int(flask.g.query_time * 1000) db_reads = flask.g.query_count db_writes = flask.g.write_count response.headers.add('Execution-Time', resp_time) response.headers.add('Query-Time', db_time) response.headers.add('Query-Count', db_reads) response.headers.add('Write-Count', db_writes) if not flask.request.path.startswith('/event'): monitoring.insert_point('requests', { 'host': settings.local.host.name, }, { 'path': flask.request.path, 'remote_ip': utils.get_remote_addr(), 'response_time': resp_time, 'db_time': db_time, 'db_reads': db_reads, 'db_writes': db_writes, }) return response
def _disconnected(self, client): org_id = client['org_id'] user_id = client['user_id'] remote_ip = client['real_address'] org = self.get_org(org_id) if org: user = org.get_user(user_id, fields=('_id', 'name')) if user: user.audit_event( 'user_connection', 'User disconnected from "%s"' % self.server.name, remote_addr=remote_ip, ) monitoring.insert_point('user_disconnections', { 'host': settings.local.host.name, 'server': self.server.name, }, { 'user': user.name, 'remote_ip': remote_ip, }) if self.route_clients: messenger.publish( 'client', { 'state': False, 'server_id': self.server.id, 'virt_address': client['virt_address'], 'virt_address6': client['virt_address6'], 'host_address': settings.local.host.local_addr, 'host_address6': settings.local.host.local_addr6, }) self.instance_com.push_output('User disconnected user_id=%s' % client['user_id']) self.send_event()
def allow_client(self, client_data, org, user, reauth=False): client_id = client_data['client_id'] key_id = client_data['key_id'] org_id = client_data['org_id'] user_id = client_data['user_id'] device_id = client_data.get('device_id') device_name = client_data.get('device_name') platform = client_data.get('platform') mac_addr = client_data.get('mac_addr') remote_ip = client_data.get('remote_ip') address_dynamic = False if reauth: doc = self.clients.find_id(client_id) if not doc: self.instance_com.send_client_deny(client_id, key_id, 'Client connection info timed out') return virt_address = doc['virt_address'] virt_address6 = doc['virt_address6'] else: user.audit_event( 'user_connection', 'User connected to "%s"' % self.server.name, remote_addr=remote_ip, ) monitoring.insert_point('user_connections', { 'host': settings.local.host.name, 'server': self.server.name, }, { 'user': user.name, 'platform': platform, 'remote_ip': remote_ip, }) virt_address = self.server.get_ip_addr(org_id, user_id) if not virt_address: logger.error('User missing ip address', 'clients', server_id=self.server.id, instance_id=self.instance.id, user_id=user.id, multi_device=self.server.multi_device, network=self.server.network, user_count=self.server.user_count, ) if not self.server.multi_device: for clnt in self.clients.find({'user_id': user_id}): time.sleep(3) self.instance_com.client_kill(clnt['id']) elif virt_address: if mac_addr: for clnt in self.clients.find({ 'user_id': user_id, 'mac_addr': mac_addr, }): self.instance_com.client_kill(clnt['id']) if self.clients.find({'virt_address': virt_address}): virt_address = None if not virt_address: while True: try: ip_addr = self.ip_pool.pop() except IndexError: break ip_addr = '%s/%s' % (ip_addr, self.ip_network.prefixlen) if not self.clients.find({'virt_address': ip_addr}): virt_address = ip_addr address_dynamic = True break if not virt_address: logger.error('Unable to assign ip address, pool full', 'clients', server_id=self.server.id, instance_id=self.instance.id, user_id=user.id, multi_device=self.server.multi_device, network=self.server.network, user_count=self.server.user_count, ) if not virt_address: self.instance_com.send_client_deny(client_id, key_id, 'Unable to assign ip address') return virt_address6 = self.server.ip4to6(virt_address) dns_servers = [] if user.dns_servers: for dns_server in user.dns_servers: if dns_server == '127.0.0.1': dns_server = virt_address dns_servers.append(dns_server) rules, rules6 = self.generate_iptables_rules( user, virt_address, virt_address6) self.clients.insert({ 'id': client_id, 'org_id': org_id, 'org_name': org.name, 'user_id': user_id, 'user_name': user.name, 'user_type': user.type, 'dns_servers': dns_servers, 'dns_suffix': user.dns_suffix, 'device_id': device_id, 'device_name': device_name, 'platform': platform, 'mac_addr': mac_addr, 'virt_address': virt_address, 'virt_address6': virt_address6, 'real_address': remote_ip, 'address_dynamic': address_dynamic, 'iptables_rules': rules, 'ip6tables_rules': rules6, }) if user.type == CERT_CLIENT: plugins.event( 'user_connected', host_id=settings.local.host_id, server_id=self.server.id, org_id=org.id, user_id=user.id, host_name=settings.local.host.name, server_name=self.server.name, org_name=org.name, user_name=user.name, platform=platform, device_id=device_id, device_name=device_name, virtual_ip=virt_address, virtual_ip6=virt_address6, remote_ip=remote_ip, mac_addr=mac_addr, ) host.global_clients.insert({ 'instance_id': self.instance.id, 'client_id': client_id, }) client_conf = self.generate_client_conf(platform, client_id, virt_address, user, reauth) client_conf += 'ifconfig-push %s %s\n' % utils.parse_network( virt_address) if self.server.ipv6: client_conf += 'ifconfig-ipv6-push %s\n' % virt_address6 if self.server.debug: self.instance_com.push_output('Client conf %s:' % user_id) for conf_line in client_conf.split('\n'): if conf_line: self.instance_com.push_output(' ' + conf_line) self.instance_com.send_client_auth(client_id, key_id, client_conf)
def _keep_alive_thread(): host_event = False last_update = None proc_stat = None settings.local.host_ping_timestamp = utils.now() cur_public_ip = None cur_public_ip6 = None cur_host_name = settings.local.host.name cur_route53_region = settings.app.route53_region cur_route53_zone = settings.app.route53_zone auto_public_host = settings.local.host.auto_public_host auto_public_host6 = settings.local.host.auto_public_host6 while True: try: timestamp = utils.now() timestamp -= datetime.timedelta( microseconds=timestamp.microsecond, seconds=timestamp.second, ) if timestamp != last_update: last_update = timestamp last_proc_stat = proc_stat proc_stat = host.usage_utils.get_proc_stat() if last_proc_stat and proc_stat: cpu_usage = host.usage_utils.calc_cpu_usage( last_proc_stat, proc_stat) mem_usage = host.usage_utils.get_mem_usage() settings.local.host.usage.add_period(timestamp, cpu_usage, mem_usage) yield interrupter_sleep(settings.app.host_ping) ping_timestamp = utils.now() try: open_file_count = len(os.listdir('/proc/self/fd')) except: open_file_count = 0 cpu_usage = None mem_usage = None thread_count = threading.active_count() server_count = len(host.global_servers) device_count = host.global_clients.count({}) try: cpu_usage, mem_usage = utils.get_process_cpu_mem() except: logger.exception('Failed to get process cpu and mem usage', 'runners', host_id=settings.local.host.id, host_name=settings.local.host.name, ) host_name = settings.local.host.name route53_region = settings.app.route53_region route53_zone = settings.app.route53_zone if route53_region and route53_zone: if cur_public_ip != settings.local.public_ip or \ cur_public_ip6 != settings.local.public_ip6 or \ cur_host_name != host_name or \ cur_route53_region != route53_region or \ cur_route53_zone != route53_zone: cur_host_name = host_name cur_public_ip = settings.local.public_ip cur_public_ip6 = settings.local.public_ip6 cur_route53_region = route53_region cur_route53_zone = route53_zone auto_public_host, auto_public_host6 = \ utils.set_zone_record( route53_region, route53_zone, host_name, cur_public_ip, cur_public_ip6, ) settings.local.host.auto_public_host = auto_public_host settings.local.host.auto_public_host6 = auto_public_host6 host_event = True else: auto_public_host = None auto_public_host6 = None if settings.local.host.auto_public_address != \ settings.local.public_ip or \ settings.local.host.auto_public_address6 != \ settings.local.public_ip6: settings.local.host.auto_public_address = \ settings.local.public_ip settings.local.host.auto_public_address6 = \ settings.local.public_ip6 host_event = True settings.local.host.collection.update({ '_id': settings.local.host.id, }, {'$set': { 'server_count': server_count, 'device_count': device_count, 'cpu_usage': cpu_usage, 'mem_usage': mem_usage, 'thread_count': thread_count, 'open_file_count': open_file_count, 'status': ONLINE, 'ping_timestamp': utils.now(), 'auto_public_address': settings.local.public_ip, 'auto_public_address6': settings.local.public_ip6, 'auto_public_host': auto_public_host, 'auto_public_host6': auto_public_host6, }}) if host_event: host_event = False event.Event(type=HOSTS_UPDATED) monitoring.insert_point('system', { 'host': settings.local.host.name, }, { 'cpu_usage': cpu_usage, 'mem_usage': mem_usage, 'thread_count': thread_count, 'open_file_count': open_file_count, }) settings.local.host_ping_timestamp = ping_timestamp except GeneratorExit: host.deinit() raise except: logger.exception('Error in host keep alive update', 'runners', host_id=settings.local.host.id, host_name=settings.local.host.name, ) time.sleep(0.5)
def allow_client(self, client_data, org, user, reauth=False): client_id = client_data['client_id'] key_id = client_data['key_id'] org_id = client_data['org_id'] user_id = client_data['user_id'] device_id = client_data.get('device_id') device_name = client_data.get('device_name') platform = client_data.get('platform') mac_addr = client_data.get('mac_addr') remote_ip = client_data.get('remote_ip') doc_id = utils.ObjectId() if reauth: doc = self.clients.find_id(client_id) if not doc: self.instance_com.send_client_deny(client_id, key_id, 'Client connection info timed out') return virt_address = doc['virt_address'] virt_address6 = doc['virt_address6'] else: user.audit_event( 'user_connection', 'User connected to "%s"' % self.server.name, remote_addr=remote_ip, ) monitoring.insert_point('user_connections', { 'host': settings.local.host.name, 'server': self.server.name, }, { 'user': user.name, 'platform': platform, 'remote_ip': remote_ip, }) virt_address, address_dynamic = self.get_virt_addr( org_id, user_id, mac_addr, doc_id) if not self.server.multi_device: if self.server.replicating: # if self.server.route_clients: # docs = self.collection.find({ # 'user_id': user_id, # 'server_id': self.server.id, # }) # # for doc in docs: # messenger.publish('client', { # 'state': False, # 'server_id': self.server.id, # 'virt_address': doc['virt_address'], # 'virt_address6': doc['virt_address6'], # 'host_address': doc['host_address'], # 'host_address6': doc['host_address6'], # }) messenger.publish('instance', [ 'user_reconnect', user_id, settings.local.host_id, ]) for clnt in self.clients.find({'user_id': user_id}): time.sleep(2) self.instance_com.client_kill(clnt['id']) if not virt_address: self.instance_com.send_client_deny(client_id, key_id, 'Unable to assign ip address') return virt_address6 = self.server.ip4to6(virt_address) dns_servers = [] if user.dns_servers: for dns_server in user.dns_servers: if dns_server == '127.0.0.1': dns_server = virt_address dns_servers.append(dns_server) rules, rules6 = self.generate_iptables_rules( user, virt_address, virt_address6) self.clients.insert({ 'id': client_id, 'doc_id': doc_id, 'org_id': org_id, 'org_name': org.name, 'user_id': user_id, 'user_name': user.name, 'user_type': user.type, 'dns_servers': dns_servers, 'dns_suffix': user.dns_suffix, 'device_id': device_id, 'device_name': device_name, 'platform': platform, 'mac_addr': mac_addr, 'virt_address': virt_address, 'virt_address6': virt_address6, 'real_address': remote_ip, 'address_dynamic': address_dynamic, 'iptables_rules': rules, 'ip6tables_rules': rules6, }) if user.type == CERT_CLIENT: plugins.event( 'user_connected', host_id=settings.local.host_id, server_id=self.server.id, org_id=org.id, user_id=user.id, host_name=settings.local.host.name, server_name=self.server.name, org_name=org.name, user_name=user.name, platform=platform, device_id=device_id, device_name=device_name, virtual_ip=virt_address, virtual_ip6=virt_address6, remote_ip=remote_ip, mac_addr=mac_addr, ) host.global_clients.insert({ 'instance_id': self.instance.id, 'client_id': client_id, }) client_conf = self.generate_client_conf(platform, client_id, virt_address, user, reauth) client_conf += 'ifconfig-push %s %s\n' % utils.parse_network( virt_address) if self.server.ipv6: client_conf += 'ifconfig-ipv6-push %s\n' % virt_address6 if self.server.debug: self.instance_com.push_output('Client conf %s:' % user_id) for conf_line in client_conf.split('\n'): if conf_line: self.instance_com.push_output(' ' + conf_line) self.instance_com.send_client_auth(client_id, key_id, client_conf)