def check_users(project, users, timeout=5): address = project.get("connection_check_address") if not address: logger.debug("no connection check address for project {0}".format( project['name'])) raise Return(()) http_client = AsyncHTTPClient() request = HTTPRequest(address, method="POST", body=urlencode( {'users': json.dumps(list(users))}), request_timeout=timeout) try: response = yield http_client.fetch(request) except Exception as err: logger.error(err) raise Return((None, None)) else: if response.code != 200: raise Return((None, None)) try: content = [str(x) for x in json.loads(response.body)] except Exception as err: logger.error(err) raise Return((None, err)) raise Return((content, None))
def check_users(project, users, timeout=5): address = project.get("connection_check_address") if not address: logger.debug("no connection check address for project {0}".format(project['name'])) raise Return(()) http_client = AsyncHTTPClient() request = HTTPRequest( address, method="POST", body=urlencode({ 'users': json.dumps(list(users)) }), request_timeout=timeout ) try: response = yield http_client.fetch(request) except Exception as err: logger.error(err) raise Return((None, None)) else: if response.code != 200: raise Return((None, None)) try: content = [str(x) for x in json.loads(response.body)] except Exception as err: logger.error(err) raise Return((None, err)) raise Return((content, None))
def authorize(self, auth_address, project, channel): """ Send POST request to web application to ask it if current client has a permission to subscribe on channel. """ project_id = self.project_id http_client = AsyncHTTPClient() request = HTTPRequest( auth_address, method="POST", body=urlencode({ 'user': self.user, 'channel': channel }), request_timeout=1 ) max_auth_attempts = project.get('max_auth_attempts') back_off_interval = project.get('back_off_interval') back_off_max_timeout = project.get('back_off_max_timeout') attempts = 0 while attempts < max_auth_attempts: # get current timeout for project current_attempts = self.application.back_off.setdefault(project_id, 0) factor = random.randint(0, 2**current_attempts-1) timeout = factor*back_off_interval if timeout > back_off_max_timeout: timeout = back_off_max_timeout # wait before next authorization request attempt yield sleep(float(timeout)/1000) try: response = yield http_client.fetch(request) except Exception as err: # let it fail and try again after some timeout # until we have auth attempts logger.debug(err) else: # reset back-off attempts self.application.back_off[project_id] = 0 if response.code == 200: # auth successful self.update_channel_user_info(response.body, channel) raise Return((True, None)) else: # access denied for this client raise Return((False, None)) attempts += 1 self.application.back_off[project_id] += 1 raise Return((False, None))
def decode_data(data): """ Decode request body received from API client. """ try: return json_decode(data) except Exception as err: logger.debug(err) return None
def post(self, project_key): """ Handle API HTTP requests. """ timer = None if self.application.collector: timer = self.application.collector.get_timer('api_time') if not self.request.body: raise tornado.web.HTTPError(400, log_message="empty request") if self.request.headers.get("Content-Type", "").startswith("application/json"): # handle JSON requests if corresponding Content-Type specified encoded_data = self.request.body sign = self.request.headers.get("X-API-Sign") else: # handle application/x-www-form-urlencoded request sign = self.get_argument('sign', None) encoded_data = self.get_argument('data', None) if not sign: raise tornado.web.HTTPError(400, log_message="no data sign") if not encoded_data: raise tornado.web.HTTPError(400, log_message="no data") project = self.application.get_project(project_key) if not project: raise tornado.web.HTTPError(404, log_message="project not found") # use project secret to validate sign secret = project['secret'] is_valid = auth.check_sign( secret, project_key, encoded_data, sign ) if not is_valid: raise tornado.web.HTTPError(401, log_message="unauthorized") try: data = json_decode(encoded_data) except Exception as err: logger.debug(err) raise tornado.web.HTTPError(400, log_message="malformed data") multi_response, error = yield self.application.process_api_data(project, data) if error: raise tornado.web.HTTPError(400, log_message=error) if self.application.collector: self.application.collector.incr('api') timer.stop() self.json_response(multi_response.as_message())
def __init__(self, sock, info): self.sock = sock self.info = info self.uid = uuid.uuid4().hex self.is_authenticated = False self.user = "" self.channel_user_info = {} self.default_user_info = {} self.project_id = None self.channels = None self.presence_ping = None logger.debug("new client created (uid: {0}, ip: {1})".format(self.uid, getattr(self.info, "ip", "-")))
def __init__(self, sock, info): self.sock = sock self.info = info self.uid = uuid.uuid4().hex self.is_authenticated = False self.user = '' self.channel_user_info = {} self.default_user_info = {} self.project_id = None self.channels = None self.presence_ping = None logger.debug("new client created (uid: {0}, ip: {1})".format( self.uid, getattr(self.info, 'ip', '-')))
def message_received(self, message): """ Called when message from client received. """ multi_response = MultiResponse() try: data = json_decode(message) except ValueError: logger.error('malformed JSON data') yield self.close_sock() raise Return((True, None)) if isinstance(data, dict): # single object request response, err = yield self.process_obj(data) multi_response.add(response) if err: # error occurred, connection must be closed logger.error(err) yield self.sock.send(multi_response.as_message()) yield self.close_sock() raise Return((True, None)) elif isinstance(data, list): # multiple object request if len(data) > self.application.CLIENT_API_MESSAGE_LIMIT: logger.debug("client API message limit exceeded") yield self.close_sock() raise Return((True, None)) for obj in data: response, err = yield self.process_obj(obj) multi_response.add(response) if err: # close connection in case of any error logger.error(err) yield self.sock.send(multi_response.as_message()) yield self.send_disconnect_message() yield self.close_sock() raise Return((True, None)) else: logger.error('data not list and not dictionary') yield self.close_sock() raise Return((True, None)) yield self.send(multi_response.as_message()) raise Return((True, None))
def handle_connect(self, params): """ Authenticate client's connection, initialize required variables in case of successful authentication. """ if self.is_authenticated: raise Return((True, None)) token = params["token"] user = params["user"] project_id = params["project"] user_info = params.get("info", None) project, error = yield self.get_project(project_id) if error: raise Return((None, error)) secret_key = project['secret_key'] if token != auth.get_client_token( secret_key, project_id, user, user_info=user_info): raise Return((None, "invalid token")) if user_info is not None: try: user_info = json_decode(user_info) except Exception as err: logger.debug("malformed JSON data in user_info") logger.debug(err) user_info = None self.is_authenticated = True self.project_id = project_id self.user = user self.default_user_info = { 'user_id': self.user, 'client_id': self.uid, 'default_info': user_info, 'channel_info': None } self.channels = {} if not self.application.state.fake: self.presence_ping = PeriodicCallback( self.send_presence_ping, self.application.state.presence_ping_interval) self.presence_ping.start() raise Return((self.uid, None))
def handle_connect(self, params): """ Authenticate client's connection, initialize required variables in case of successful authentication. """ if self.is_authenticated: raise Return((True, None)) token = params["token"] user = params["user"] project_id = params["project"] user_info = params.get("info", None) project, error = yield self.get_project(project_id) if error: raise Return((None, error)) secret_key = project['secret_key'] if token != auth.get_client_token(secret_key, project_id, user, user_info=user_info): raise Return((None, "invalid token")) if user_info is not None: try: user_info = json_decode(user_info) except Exception as err: logger.debug("malformed JSON data in user_info") logger.debug(err) user_info = None self.is_authenticated = True self.project_id = project_id self.user = user self.default_user_info = { 'user_id': self.user, 'client_id': self.uid, 'default_info': user_info, 'channel_info': None } self.channels = {} if not self.application.state.fake: self.presence_ping = PeriodicCallback( self.send_presence_ping, self.application.state.presence_ping_interval ) self.presence_ping.start() raise Return((self.uid, None))
def __init__(self, sock, info): self.sock = sock self.info = info self.uid = uuid.uuid4().hex self.is_authenticated = False self.user = None self.token = None self.examined_at = None self.channel_user_info = {} self.default_user_info = {} self.project_id = None self.channels = None self.presence_ping_task = None self.connect_queue = None logger.debug("new client created (uid: {0}, ip: {1})".format( self.uid, getattr(self.info, 'ip', '-') ))
def update(self): """ Call this method periodically to keep structure consistency """ if not self.storage: raise Return((True, None)) with (yield lock.acquire()): raw_projects, error = yield self.storage.project_list() if error: self.on_error(error) projects = [flatten(x) for x in raw_projects] raw_namespaces, error = yield self.storage.namespace_list() if error: self.on_error(error) namespaces = [flatten(x) for x in raw_namespaces] projects_by_id = get_projects_by_id(projects) projects_by_name = get_projects_by_name(projects) namespaces_by_id = get_namespaces_by_id(namespaces) namespaces_by_name = get_namespaces_by_name(namespaces) project_namespaces = get_project_namespaces(namespaces) self._data = { 'projects': projects, 'namespaces': namespaces, 'projects_by_id': projects_by_id, 'projects_by_name': projects_by_name, 'namespaces_by_id': namespaces_by_id, 'namespaces_by_name': namespaces_by_name, 'project_namespaces': project_namespaces } self._consistent = True self.structure_recover.stop() logger.debug('Structure updated') raise Return((True, None))
def update(self): """ Call this method periodically to keep structure consistency """ if not self.storage or not self.db: raise Return((True, None)) with (yield lock.acquire()): projects, error = yield self.storage.project_list(self.db) if error: self.on_error(error) namespaces, error = yield self.storage.namespace_list(self.db) if error: self.on_error(error) projects_by_id = self.storage.projects_by_id(projects) projects_by_name = self.storage.projects_by_name(projects) namespaces_by_id = self.storage.namespaces_by_id(namespaces) namespaces_by_name = self.storage.namespaces_by_name(namespaces) project_namespaces = self.storage.project_namespaces(namespaces) self._data = { 'projects': projects, 'namespaces': namespaces, 'projects_by_id': projects_by_id, 'projects_by_name': projects_by_name, 'namespaces_by_id': namespaces_by_id, 'namespaces_by_name': namespaces_by_name, 'project_namespaces': project_namespaces } self._consistent = True logger.debug('Structure updated') raise Return((True, None))
def update(self): """ Call this method periodically to keep structure consistency """ if not self.storage or not self.db: raise Return((True, None)) with (yield lock.acquire()): projects, error = yield self.storage.project_list(self.db) if error: self.on_error(error) namespaces, error = yield self.storage.namespace_list(self.db) if error: self.on_error(error) projects_by_id = self.storage.projects_by_id(projects) projects_by_name = self.storage.projects_by_name(projects) namespaces_by_id = self.storage.namespaces_by_id(namespaces) namespaces_by_name = self.storage.namespaces_by_name(namespaces) project_namespaces = self.storage.project_namespaces(namespaces) self._data = { 'projects': projects, 'namespaces': namespaces, 'projects_by_id': projects_by_id, 'projects_by_name': projects_by_name, 'namespaces_by_id': namespaces_by_id, 'namespaces_by_name': namespaces_by_name, 'project_namespaces': project_namespaces } self._CONSISTENT = True logger.debug('Structure updated') raise Return((True, None))
def subscribe(self): self.uid = uuid.uuid4().hex self.application.add_admin_connection(self.uid, self) logger.debug('admin subscribed')
def create_centrifuge_application(): try: custom_settings = json.load(open(options.config, 'r')) except IOError: logger.warning("No configuration file found. " "In production make sure security settings provided") custom_settings = {} # override security related options using environment variable # value if exists for option_name in ["password", "cookie_secret", "api_secret"]: environment_var_name = "CENTRIFUGE_{0}".format(option_name.upper()) environment_value = os.environ.get(environment_var_name) if environment_value: logger.debug( "using {0} environment variable for {1} option value".format( environment_var_name, option_name)) custom_settings[option_name] = environment_value if os.environ.get("CENTRIFUGE_INSECURE") == "1": custom_settings["insecure"] = True settings = dict( cookie_secret=custom_settings.get("cookie_secret", "bad secret"), login_url="/auth", template_path=os.path.join(os.path.dirname(__file__), os.path.join("web/frontend", "templates")), static_path=os.path.join(os.path.dirname(__file__), os.path.join("web/frontend", "static")), xsrf_cookies=True, autoescape="xhtml_escape", debug=options.debug, options=options, config=custom_settings) sockjs_settings = custom_settings.get("sockjs_settings", {}) if not sockjs_settings or not sockjs_settings.get("sockjs_url"): # SockJS CDN will be retired # see https://github.com/sockjs/sockjs-client/issues/198 # if no explicit SockJS url provided in configuration file # then we use jsdelivr CDN instead of default cdn.sockjs.org # this can be fixed directly in SockJS-Tornado soon sockjs_settings[ "sockjs_url"] = "https://cdn.jsdelivr.net/sockjs/0.3/sockjs.min.js" handlers = create_application_handlers(sockjs_settings) # custom settings to configure the tornado HTTPServer tornado_settings = custom_settings.get("tornado_settings", {}) logger.debug("tornado_settings: %s", tornado_settings) if 'io_loop' in tornado_settings: stop_running( "The io_loop in tornado_settings is not supported for now.") try: app = Application(handlers=handlers, **settings) server = tornado.httpserver.HTTPServer(app, **tornado_settings) server.listen(options.port, address=options.address) except Exception as e: return stop_running(str(e)) logger.info("Engine class: {0}".format(engine_class_path)) app.engine = engine_class(app) logger.info("Storage class: {0}".format(storage_class_path)) app.storage = storage_class(options) # create reference to application from SockJS handlers AdminSocketHandler.application = app # create reference to application from Client Client.application = app app.initialize() logger.info("Tornado port: {0}, address: {1}".format( options.port, options.address)) return app
def handle_connect(self, params): """ Authenticate client's connection, initialize required variables in case of successful authentication. """ if self.application.collector: self.application.collector.incr('connect') if self.is_authenticated: raise Return((self.uid, None)) token = params["token"] user = params["user"] project_id = params["project"] timestamp = params["timestamp"] user_info = params.get("info") project, error = yield self.application.get_project(project_id) if error: raise Return((None, error)) secret_key = project['secret_key'] try: client_token = auth.get_client_token(secret_key, project_id, user, timestamp, user_info=user_info) except Exception as err: logger.error(err) raise Return((None, "invalid connection parameters")) if token != client_token: raise Return((None, "invalid token")) if user_info is not None: try: user_info = json_decode(user_info) except Exception as err: logger.debug("malformed JSON data in user_info") logger.debug(err) user_info = None try: timestamp = int(timestamp) except ValueError: raise Return((None, "invalid timestamp")) now = time.time() self.user = user self.examined_at = timestamp connection_check = project.get('connection_check', False) if connection_check and self.examined_at + project.get("connection_lifetime", 24*365*3600) < now: # connection expired - this is a rare case when Centrifuge went offline # for a while or client turned on his computer from sleeping mode. # put this client into the queue of connections waiting for # permission to reconnect with expired credentials. To avoid waiting # client must reconnect with actual credentials i.e. reload browser # window. if project_id not in self.application.expired_reconnections: self.application.expired_reconnections[project_id] = [] self.application.expired_reconnections[project_id].append(self) if project_id not in self.application.expired_connections: self.application.expired_connections[project_id] = { "users": set(), "checked_at": None } self.application.expired_connections[project_id]["users"].add(user) self.connect_queue = toro.Queue(maxsize=1) value = yield self.connect_queue.get() if not value: yield self.close_sock() raise Return((None, self.application.UNAUTHORIZED)) else: self.connect_queue = None # Welcome to Centrifuge dear Connection! self.is_authenticated = True self.project_id = project_id self.token = token self.default_user_info = { 'user_id': self.user, 'client_id': self.uid, 'default_info': user_info, 'channel_info': None } self.channels = {} self.presence_ping_task = PeriodicCallback( self.send_presence_ping, self.application.engine.presence_ping_interval ) self.presence_ping_task.start() self.application.add_connection(project_id, self.user, self.uid, self) raise Return((self.uid, None))
def unsubscribe(self): if not hasattr(self, 'uid'): return self.application.remove_admin_connection(self.uid) logger.debug('admin disconnected')
def close(self): yield self.clean() logger.debug('client destroyed (uid: %s)' % self.uid) raise Return((True, None))
def unsubscribe(self): if not self.uid: return self.application.remove_admin_connection(self.uid) logger.debug('admin unsubscribed')
def post(self, project_id): """ Handle API HTTP requests. """ timer = None if self.application.collector: timer = self.application.collector.get_timer('api_time') if not self.request.body: raise tornado.web.HTTPError(400, log_message="empty request") if self.request.headers.get("Content-Type", "").startswith("application/json"): # handle JSON requests if corresponding Content-Type specified try: request_data = json_decode(self.request.body) except ValueError: raise tornado.web.HTTPError(400, log_message="malformed json") if not isinstance(request_data, dict): raise tornado.web.HTTPError(400, log_message="object expected") sign = request_data.get("sign") encoded_data = request_data.get("data") else: # handle application/x-www-form-urlencoded request sign = self.get_argument('sign', None) encoded_data = self.get_argument('data', None) if not sign: raise tornado.web.HTTPError(400, log_message="no data sign") if not encoded_data: raise tornado.web.HTTPError(400, log_message="no data") is_owner_request = False if project_id == self.application.OWNER_API_PROJECT_ID: # API request aims to be from superuser is_owner_request = True if is_owner_request: # use api secret key from configuration to check sign secret = self.application.config.get("api_secret") if not secret: raise tornado.web.HTTPError( 501, log_message="no api_secret in configuration file") project = None else: project, error = yield self.application.structure.get_project_by_id( project_id) if error: raise tornado.web.HTTPError(500, log_message=str(error)) if not project: raise tornado.web.HTTPError(404, log_message="project not found") # use project secret key to validate sign secret = project['secret_key'] is_valid = auth.check_sign(secret, project_id, encoded_data, sign) if not is_valid: raise tornado.web.HTTPError(401, log_message="unauthorized") try: data = json_decode(encoded_data) except Exception as err: logger.debug(err) raise tornado.web.HTTPError(400, log_message="malformed data") multi_response, error = yield self.application.process_api_data( project, data, is_owner_request) if error: raise tornado.web.HTTPError(400, log_message=error) if self.application.collector: self.application.collector.incr('api') timer.stop() self.json_response(multi_response.as_message())
def post(self, project_id): """ Handle API HTTP requests. """ timer = None if self.application.collector: timer = self.application.collector.get_timer('api_time') if not self.request.body: raise tornado.web.HTTPError(400, log_message="empty request") if self.request.headers.get("Content-Type", "").startswith("application/json"): # handle JSON requests if corresponding Content-Type specified try: request_data = json_decode(self.request.body) except ValueError: raise tornado.web.HTTPError(400, log_message="malformed json") if not isinstance(request_data, dict): raise tornado.web.HTTPError(400, log_message="object expected") sign = request_data.get("sign") encoded_data = request_data.get("data") else: # handle application/x-www-form-urlencoded request sign = self.get_argument('sign', None) encoded_data = self.get_argument('data', None) if not sign: raise tornado.web.HTTPError(400, log_message="no data sign") if not encoded_data: raise tornado.web.HTTPError(400, log_message="no data") is_owner_request = False if project_id == self.application.OWNER_API_PROJECT_ID: # API request aims to be from superuser is_owner_request = True if is_owner_request: # use api secret key from configuration to check sign secret = self.application.config.get("api_secret") if not secret: raise tornado.web.HTTPError(501, log_message="no api_secret in configuration file") project = None else: project, error = yield self.application.structure.get_project_by_id(project_id) if error: raise tornado.web.HTTPError(500, log_message=str(error)) if not project: raise tornado.web.HTTPError(404, log_message="project not found") # use project secret key to validate sign secret = project['secret_key'] is_valid = auth.check_sign( secret, project_id, encoded_data, sign ) if not is_valid: raise tornado.web.HTTPError(401, log_message="unauthorized") try: data = json_decode(encoded_data) except Exception as err: logger.debug(err) raise tornado.web.HTTPError(400, log_message="malformed data") multi_response, error = yield self.application.process_api_data(project, data, is_owner_request) if error: raise tornado.web.HTTPError(400, log_message=error) if self.application.collector: self.application.collector.incr('api') timer.stop() self.json_response(multi_response.as_message())
def create_centrifuge_application(): try: custom_settings = json.load(open(options.config, 'r')) except IOError: return stop_running("No configuration file found.") # override security related options using environment variable # value if exists for option_name in ["password", "cookie_secret", "api_secret"]: environment_var_name = "CENTRIFUGE_{0}".format(option_name.upper()) environment_value = os.environ.get(environment_var_name) if environment_value: logger.debug("using {0} environment variable for {1} option value".format( environment_var_name, option_name )) custom_settings[option_name] = environment_value if os.environ.get("CENTRIFUGE_INSECURE") == "1": custom_settings["insecure"] = True settings = dict( cookie_secret=custom_settings.get("cookie_secret", "bad secret"), template_path=os.path.join( os.path.dirname(__file__), os.path.join("web/frontend", "templates") ), static_path=os.path.join( os.path.dirname(__file__), os.path.join("web/frontend", "static") ), xsrf_cookies=False, autoescape="xhtml_escape", debug=options.debug, options=options, config=custom_settings ) sockjs_settings = custom_settings.get("sockjs_settings", {}) if not sockjs_settings or not sockjs_settings.get("sockjs_url"): # SockJS CDN will be retired # see https://github.com/sockjs/sockjs-client/issues/198 # if no explicit SockJS url provided in configuration file # then we use jsdelivr CDN instead of default cdn.sockjs.org # this can be fixed directly in SockJS-Tornado soon sockjs_settings["sockjs_url"] = "https://cdn.jsdelivr.net/sockjs/1.0/sockjs.min.js" handlers = create_application_handlers(sockjs_settings) # custom settings to configure the tornado HTTPServer tornado_settings = custom_settings.get("tornado_settings", {}) logger.debug("tornado_settings: %s", tornado_settings) if 'io_loop' in tornado_settings: stop_running( "The io_loop in tornado_settings is not supported for now." ) try: app = Application(handlers=handlers, **settings) server = tornado.httpserver.HTTPServer(app, **tornado_settings) server.listen(options.port, address=options.address) except Exception as e: return stop_running(str(e)) logger.info("Engine class: {0}".format(engine_class_path)) app.engine = engine_class(app) # create reference to application from Client Client.application = app app.initialize() logger.info("Tornado port: {0}, address: {1}".format(options.port, options.address)) return app
def authorize(self, auth_address, project, namespace_name, channel): """ Send POST request to web application to ask it if current client has a permission to subscribe on channel. """ project_id = self.project_id http_client = AsyncHTTPClient() request = HTTPRequest(auth_address, method="POST", body=urlencode({ 'user': self.user, 'namespace': namespace_name, 'channel': channel }), request_timeout=1) max_auth_attempts = project.get('max_auth_attempts') back_off_interval = project.get('back_off_interval') back_off_max_timeout = project.get('back_off_max_timeout') attempts = 0 while attempts < max_auth_attempts: # get current timeout for project current_attempts = self.application.back_off.setdefault( project_id, 0) factor = random.randint(0, 2**current_attempts - 1) timeout = factor * back_off_interval if timeout > back_off_max_timeout: timeout = back_off_max_timeout # wait before next authorization request attempt yield sleep(float(timeout) / 1000) try: response = yield http_client.fetch(request) except Exception as e: # let it fail and try again after some timeout # until we have auth attempts logger.debug(e) else: # reset back-off attempts self.application.back_off[project_id] = 0 if response.code == 200: # auth successful self.update_channel_user_info(response.body, namespace_name, channel) raise Return((True, None)) elif response.code == 403: # access denied for this client raise Return((False, None)) attempts += 1 self.application.back_off[project_id] += 1 raise Return((False, None))