Ejemplo n.º 1
0
    def _api_request(self, method, url, **kwargs):
        """Make an API request"""
        allow_404 = kwargs.pop('allow_404', False)
        headers = kwargs.setdefault('headers', {})
        headers.setdefault('Authorization', 'token %s' % self.api_token)
        try:
            r = requests.request(method, url, **kwargs)
        except requests.ConnectionError as e:
            app_log.error("Error connecting to %s: %s", self.api_url, e)
            msg = "Failed to connect to Hub API at %r." % self.api_url
            msg += "  Is the Hub accessible at this URL (from host: %s)?" % socket.gethostname()
            if '127.0.0.1' in self.api_url:
                msg += "  Make sure to set c.JupyterHub.hub_ip to an IP accessible to" + \
                       " single-user servers if the servers are not on the same host as the Hub."
            raise HTTPError(500, msg)

        data = None
        if r.status_code == 404 and allow_404:
            pass
        elif r.status_code == 403:
            app_log.error("I don't have permission to check authorization with JupyterHub, my auth token may have expired: [%i] %s", r.status_code, r.reason)
            app_log.error(r.text)
            raise HTTPError(500, "Permission failure checking authorization, I may need a new token")
        elif r.status_code >= 500:
            app_log.error("Upstream failure verifying auth token: [%i] %s", r.status_code, r.reason)
            app_log.error(r.text)
            raise HTTPError(502, "Failed to check authorization (upstream problem)")
        elif r.status_code >= 400:
            app_log.warning("Failed to check authorization: [%i] %s", r.status_code, r.reason)
            app_log.warning(r.text)
            raise HTTPError(500, "Failed to check authorization")
        else:
            data = r.json()

        return data
Ejemplo n.º 2
0
    def check(self, handler):
        """Check the rate limit for a handler.
        
        Identifies the source by ip and user-agent.
        
        If the rate limit is exceeded, raise HTTPError(429)
        """
        if not self.limit:
            return
        key = self.key_for_handler(handler)
        added = yield self.cache.add(key, 1, self.interval)
        if not added:
            # it's been seen before, use incr
            try:
                count = yield self.cache.incr(key)
            except Exception as e:
                app_log.warning("Failed to increment rate limit for %s", key)
                return

            app_log.debug("Rate limit remaining for %r: %s/%s", key, self.limit - count, self.limit)

            if count and count >= self.limit:
                minutes = self.interval // 60
                raise HTTPError(429,
                    "Rate limit exceeded for {ip} ({limit} req / {minutes} min)."
                    " Try again later.".format(
                        ip=handler.request.remote_ip,
                        limit=self.limit,
                        minutes=minutes,
                    )
                )
Ejemplo n.º 3
0
    def get(self, path=None):
        '''Spawns a brand new server'''

        try:
            if path is None:
                # No path. Assign a prelaunched container from the pool and redirect to it.
                # Append self.redirect_uri to the redirect target.
                container_path = self.pool.acquire().path
                app_log.info("Allocated [%s] from the pool.", container_path)

                url = "/{}/{}".format(container_path, self.redirect_uri)
            else:
                path_parts = path.lstrip('/').split('/', 1)
                container_path = path_parts[0]

                # Scrap a container from the pool and replace it with an ad-hoc replacement.
                # This takes longer, but is necessary to support ad-hoc containers
                yield self.pool.adhoc(container_path)

                app_log.info("Allocated ad-hoc container at [%s].", container_path)
                url = path

            app_log.debug("Redirecting [%s] -> [%s].", self.request.path, url)
            self.redirect(url, permanent=False)
        except spawnpool.EmptyPoolError:
            app_log.warning("The container pool is empty!")
            self.render("full.html", cull_period=self.cull_period)
Ejemplo n.º 4
0
    def get(self, path=None):
        '''Spawns a brand new server'''
        if self.allow_origin:
            self.set_header("Access-Control-Allow-Origin", self.allow_origin)
        try:
            if path is None:
                # No path. Assign a prelaunched container from the pool and redirect to it.
                # Append self.redirect_uri to the redirect target.
                container_path = self.pool.acquire().path
                app_log.info("Allocated [%s] from the pool.", container_path)

                url = "/{}/{}".format(container_path, self.redirect_uri)
            else:
                # Split /user/{some_user}/long/url/path and acquire {some_user}
                path_parts = path.lstrip('/').split('/', 2)
                app_log.info("path parts: %s", path_parts)
                user = path_parts[1]

                # Scrap a container from the pool and replace it with an ad-hoc replacement.
                # This takes longer, but is necessary to support ad-hoc containers
                yield self.pool.adhoc(user, path_parts[-1])

                url = "/" + "/".join(path_parts[:2])
                app_log.info("new url: %s", url)

            app_log.debug("Redirecting [%s] -> [%s].", self.request.path, url)
            self.redirect(url, permanent=False)
        except spawnpool.EmptyPoolError:
            app_log.warning("The container pool is empty!")
            self.render("full.html", cull_period=self.cull_period)
Ejemplo n.º 5
0
def wait_for_http_server(url, timeout=10):
    """Wait for an HTTP Server to respond at url.

    Any non-5XX response code will do, even 404.
    """
    loop = ioloop.IOLoop.current()
    tic = loop.time()
    client = AsyncHTTPClient()
    while loop.time() - tic < timeout:
        try:
            r = yield client.fetch(url, follow_redirects=False)
        except HTTPError as e:
            if e.code >= 500:
                # failed to respond properly, wait and try again
                if e.code != 599:
                    # we expect 599 for no connection,
                    # but 502 or other proxy error is conceivable
                    app_log.warning(
                        "Server at %s responded with error: %s", url, e.code)
                yield gen.sleep(0.1)
            else:
                app_log.debug("Server at %s responded with %s", url, e.code)
                return
        except (OSError, socket.error) as e:
            if e.errno not in {errno.ECONNABORTED, errno.ECONNREFUSED, errno.ECONNRESET}:
                app_log.warning("Failed to connect to %s (%s)", url, e)
            yield gen.sleep(0.1)
        else:
            return

    raise TimeoutError(
        "Server at {url} didn't respond in {timeout} seconds".format(**locals())
    )
Ejemplo n.º 6
0
    def check_hub_user(self, user_model):
        """Check whether Hub-authenticated user should be allowed.

        Returns the input if the user should be allowed, None otherwise.

        Override if you want to check anything other than the username's presence in hub_users list.

        Args:
            user_model (dict): the user model returned from :class:`HubAuth`
        Returns:
            user_model (dict): The user model if the user should be allowed, None otherwise.
        """
        if self.hub_users is None and self.hub_groups is None:
            # no whitelist specified, allow any authenticated Hub user
            return user_model
        name = user_model['name']
        if self.hub_users and name in self.hub_users:
            # user in whitelist
            return user_model
        elif self.hub_groups and set(user_model['groups']).union(self.hub_groups):
            # group in whitelist
            return user_model
        else:
            app_log.warning("Not allowing Hub user %s" % name)
            return None
Ejemplo n.º 7
0
    def get(self):
        error = self.get_argument("error", False)
        if error:
            msg = self.get_argument("error_description", error)
            raise HTTPError(400, "Error in oauth: %s" % msg)

        code = self.get_argument("code", False)
        if not code:
            raise HTTPError(400, "oauth callback made without a token")

        # validate OAuth state
        arg_state = self.get_argument("state", None)
        if arg_state is None:
            raise HTTPError("oauth state is missing. Try logging in again.")
        cookie_name = self.hub_auth.get_state_cookie_name(arg_state)
        cookie_state = self.get_secure_cookie(cookie_name)
        # clear cookie state now that we've consumed it
        self.clear_cookie(cookie_name, path=self.hub_auth.base_url)
        if isinstance(cookie_state, bytes):
            cookie_state = cookie_state.decode('ascii', 'replace')
        # check that state matches
        if arg_state != cookie_state:
            app_log.warning("oauth state %r != %r", arg_state, cookie_state)
            raise HTTPError(403, "oauth state does not match. Try logging in again.")
        next_url = self.hub_auth.get_next_url(cookie_state)
        # TODO: make async (in a Thread?)
        token = self.hub_auth.token_for_code(code)
        session_id = self.hub_auth.get_session_id(self)
        user_model = self.hub_auth.user_for_token(token, session_id=session_id)
        if user_model is None:
            raise HTTPError(500, "oauth callback failed to identify a user")
        app_log.info("Logged-in user %s", user_model)
        self.hub_auth.set_cookie(self, token)
        self.redirect(next_url or self.hub_auth.base_url)
Ejemplo n.º 8
0
    def _check_hub_authorization(self, url, cache_key=None, use_cache=True):
        """Identify a user with the Hub
        
        Args:
            url (str): The API URL to check the Hub for authorization
                       (e.g. http://127.0.0.1:8081/hub/api/authorizations/token/abc-def)
            cache_key (str): The key for checking the cache
            use_cache (bool): Specify use_cache=False to skip cached cookie values (default: True)

        Returns:
            user_model (dict): The user model, if a user is identified, None if authentication fails.

        Raises an HTTPError if the request failed for a reason other than no such user.
        """
        if use_cache:
            if cache_key is None:
                raise ValueError("cache_key is required when using cache")
            # check for a cached reply, so we don't check with the Hub if we don't have to
            try:
                return self.cache[cache_key]
            except KeyError:
                app_log.debug("HubAuth cache miss: %s", cache_key)

        data = self._api_request('GET', url, allow_404=True)
        if data is None:
            app_log.warning("No Hub user identified for request")
        else:
            app_log.debug("Received request from Hub user %s", data)
        if use_cache:
            # cache result
            self.cache[cache_key] = data
        return data
Ejemplo n.º 9
0
	def add_error_message_to_slug(self, error_string, args_to_delete=[], msg_categ="error" ) : 
		""" add an "error" arg to url slug """

		slug_ = self.request.arguments
		app_log.info("... add_error_message_to_slug / slug_ : \n %s ", pformat( slug_ ) )

		# create a complete clean slug if no slug
		if slug_ == {} :
			error_slug = u"?" + u"error=" + tornado.escape.url_escape(error_string)
		
		# add error arg to existing slug
		else : 

			slug_without_error = deepcopy(slug_)			

			### clean existing slug from existing error arg if any
			# for arg_to_delete in args_to_delete + DEFAULT_ERROR_ARGS_TO_DELETE : 
			# 	try : 
			# 		del slug_without_error[arg_to_delete]
			# 	except :
			# 		pass
			slug_without_error = self.clean_slug(slug_without_error, args_to_delete + DEFAULT_ERROR_ARGS_TO_DELETE )

			app_log.warning("... add_error_message_to_slug / slug_without_error : \n %s ", pformat(slug_without_error) )

			# recreate slug
			error_dict	= { "error" : error_string }
			error_dict.update( slug_without_error )
			error_slug = u"?" + urllib.urlencode( error_dict, doseq=True)		

		app_log.info("... add_error_message_to_slug / error_slug : \n %s ", pformat(error_slug) )

		return error_slug
Ejemplo n.º 10
0
    def get_company_exports_summary(self, org_id):
        db_ce = self.application.mysql_db_name("company-exports")
        if not db_ce:
            app_log.warning(
                "Cannot add '%s' data. Database `%s` not online.",
                "influence", db_ce)
            return

        data = {}

        sql = """
select
    {db_ce}.company.company_id,
    count(distinct({db_ce}.action.destination_id))
  from {db_ce}.action
    join {db_ce}.company using (company_id)
  where {db_ce}.company.caat_id = {caat_id}
;
""".format(**{
    "db_ce": db_ce,
    "caat_id": org_id
})

        try:
            result = self.orm.execute(sql).fetchone()
        except InternalError as e:
            print(e)
            return

        if result:
            (company_id, n_dest, ) = result
            data["companyExportsId"] = company_id
            data["destinationCount"] = n_dest

        sql = """
select
    count(distinct({db_ce}.rating.description))
  from {db_ce}.action
    join {db_ce}.company using (company_id)
    join {db_ce}.rating using (rating_id)
  where {db_ce}.company.caat_id = {caat_id}
    and {db_ce}.rating.description is not null
;
""".format(**{
    "db_ce": db_ce,
    "caat_id": org_id
})

        try:
            result = self.orm.execute(sql).fetchone()
        except InternalError as e:
            print(e)
            return

        if result:
            (n_rating, ) = result
            data["ratingCount"] = n_rating

        return data
Ejemplo n.º 11
0
 def _get_user_cookie(self, handler):
     token = handler.get_secure_cookie(self.cookie_name)
     if token:
         user_model = self.user_for_token(token)
         if user_model is None:
             app_log.warning("Token stored in cookie may have expired")
             handler.clear_cookie(self.cookie_name)
         return user_model
Ejemplo n.º 12
0
    def observe(self):
        '''Collect Ground Truth of what's actually running from Docker and the proxy.'''

        results = yield {
            "docker": self.spawner.list_notebook_servers(self.name_pattern, all=True),
            "proxy": self._proxy_routes()
        }

        self.container_ids = set()
        self.living_container_ids = []
        self.stopped_container_ids = []
        self.zombie_container_ids = []

        self.routes = set()
        self.live_routes = []
        self.stale_routes = []
        self.zombie_routes = []

        # Sort Docker results into living and dead containers.
        for container in results["docker"]:
            id = container['Id']
            self.container_ids.add(id)
            if container['Status'].startswith('Up'):
                self.living_container_ids.append(id)
            else:
                self.stopped_container_ids.append(id)

        now = datetime.utcnow()
        idle_cutoff = now - self.cull_idle
        started_cutoff = now - self.cull_max_age

        # Sort proxy routes into living, stale, and zombie routes.
        living_set = set(self.living_container_ids)
        for path, route in results["proxy"].items():
            last_activity_s = route.get('last_activity', None)
            container_id = route.get('container_id', None)
            if container_id:
                result = (path, container_id)
                if container_id in living_set:
                    try:
                        last_activity = datetime.strptime(last_activity_s, _date_fmt)
                        started = self.started.get(container_id, None)
                        self.routes.add(result)
                        if started and last_activity < idle_cutoff:
                            app_log.info("Culling %s, idle since %s", path, last_activity)
                            self.stale_routes.append(result)
                        elif started and started < started_cutoff:
                            app_log.info("Culling %s, up since %s", path, started)
                            self.stale_routes.append(result)
                        else:
                            app_log.debug("Container %s up since %s, idle since %s",
                                          path, started, last_activity)
                            self.live_routes.append(result)
                    except ValueError as e:
                        app_log.warning("Ignoring a proxy route with an unparsable activity date: %s", e)
                else:
                    # The container doesn't correspond to a living container.
                    self.zombie_routes.append(result)
Ejemplo n.º 13
0
 def _parse_json_arguments(self):
     
     content_type = self.content_type
     
     if(content_type and content_type.find(r'application/json') >= 0):
         try:
             self.json_arguments = json_decode(self.request.body)
         except Exception as error:
             app_log.warning('Invalid application/json body: %s', error)
Ejemplo n.º 14
0
	def catch_error_message (self):
		""" get and log error message if any """
		
		try:
			self.error_msg = self.get_argument("error")
			app_log.warning("\n... get_error_message / self.error_msg : %s ", self.error_msg )
		
		except:
			self.error_msg = ""
Ejemplo n.º 15
0
 def post(self):
     '''Spawns a brand new server programatically'''
     try:
         url = self.pool.acquire().path
         app_log.info("Allocated [%s] from the pool.", url)
         app_log.debug("Responding with container url [%s].", url)
         self.write({'url': url})
     except spawnpool.EmptyPoolError:
         app_log.warning("The container pool is empty!")
         self.write({'status': 'full'})
Ejemplo n.º 16
0
 def _get_user_cookie(self, handler):
     token = handler.get_secure_cookie(self.cookie_name)
     session_id = self.get_session_id(handler)
     if token:
         token = token.decode('ascii', 'replace')
         user_model = self.user_for_token(token, session_id=session_id)
         if user_model is None:
             app_log.warning("Token stored in cookie may have expired")
             handler.clear_cookie(self.cookie_name)
         return user_model
Ejemplo n.º 17
0
 def find(cls, db, token):
     orm_token = super().find(db, token)
     if orm_token and not orm_token.client_id:
         app_log.warning(
             "Deleting stale oauth token for %s with no client",
             orm_token.user and orm_token.user.name,
         )
         db.delete(orm_token)
         db.commit()
         return
     return orm_token
Ejemplo n.º 18
0
    def _register_handlers(self, spec, handlers):
        """
        """

        handlers.append(spec)
        if spec.name:
            if spec.name in self.named_handlers:
                app_log.warning(
                    "Multiple handlers named %s; replacing previous value",
                    spec.name)
            self.named_handlers[spec.name] = spec
Ejemplo n.º 19
0
    def process_rule(self, rule: "Rule") -> "Rule":
        rule = super(ReversibleRuleRouter, self).process_rule(rule)

        if rule.name:
            if rule.name in self.named_rules:
                app_log.warning(
                    "Multiple handlers named %s; replacing previous value", rule.name
                )
            self.named_rules[rule.name] = rule

        return rule
Ejemplo n.º 20
0
 async def emit(self, data):
     if type(data) is not str:
         serialized_data = json.dumps(data)
     else:
         serialized_data = data
     try:
         self.write('data: {}\n\n'.format(serialized_data))
         await self.flush()
     except StreamClosedError:
         app_log.warning("Stream closed while handling %s", self.request.uri)
         # raise Finish to halt the handler
         raise web.Finish()
Ejemplo n.º 21
0
    def _json_default(self, obj):
        """encode non-jsonable objects as JSON

        Currently only bytes are supported

        """
        if not isinstance(obj, bytes):
            app_log.warning(
                "Non-jsonable data in user_options: %r; will persist None.", type(obj)
            )
            return None

        return {"__jupyterhub_bytes__": True, "data": encodebytes(obj).decode('ascii')}
Ejemplo n.º 22
0
    def post(self):
        self.content_type = 'application/json'

        try:
            data = json.loads(self.request.body)
            identity = data["mpinId"]
            userid = data["userId"]
            expireTime = data["expireTime"]
            activateKey = data["activateKey"]
            mobile = data["mobile"]
        except ValueError:
            log.error("Cannot decode body as JSON.")
            log.debug(self.request.body)
            self.set_status(400, reason="BAD REQUEST. INVALID JSON")
            self.finish()
            return

        userId = data.get("userId")
        if not userId:
            log.error("Missing userId")
            log.debug(self.request.body)
            self.set_status(400, reason="BAD REQUEST. INVALID USER ID")
            self.finish()
            return

        if options.verifyIdentityURL.startswith("/"):  # relative path
            base_url = "{0}/{1}".format(
                self.request.headers.get("RPS-BASE-URL").rstrip("/"),
                options.verifyIdentityURL.lstrip("/")
            )
        else:
            base_url = options.verifyIdentityURL

        validateURL = self._generateValidationURL(base_url, identity, activateKey, expireTime)
        log.info("Sending activation email for user {0}: {1}".format(userid.encode("utf-8"), validateURL))

        deviceName = mobile and "Mobile" or "PC"

        if options.forceActivate:
            log.warning("forceActivate option set! User activated without verification!")
        else:
            mailer.sendActivationEmail(userid.encode("utf-8"), options.emailSubject, deviceName, validateURL, options.smtpUser, options.smtpPassword)
            log.warning("Sending Mail!")

        responseData = {
            "forceActivate": options.forceActivate
        }
        self.write(json.dumps(responseData))

        self.set_status(200)
        self.finish()
Ejemplo n.º 23
0
    def fetch(self):
        q = list()

        on_message = self.filter(self.container)

        while len(self.__buff) > 8:
            header, self.__buff = self.__buff[:8], self.__buff[8:]
            stream, length = self.parse_header(header)

            if len(self.__buff) < length:
                continue

            if not length:
                continue

            message = self.split(length)

            ts, message = message.split(b(' '), 1)

            result = on_message.send(message)

            if result is False:
                continue
            else:
                msg = {'stream': self.STREAMS[stream], 'timestamp': ts.lstrip(b('[')).rstrip(b(']'))}

                msg.update({
                    'container': self.container.name,
                    'image': self.container.image
                })

                if result is None:
                    log.warning('Message "%s" not parsed by regular exception', message)
                    msg.update({'message': message})
                else:
                    msg.update(result)

                log.debug("Storing message %s", LasyJSON(msg))

                q.append((
                    dumps({
                        'index': {
                            '_index': self.get_index_name(),
                            '_type': 'logs',
                        }
                    }),
                    dumps(msg)
                ))

        return q
Ejemplo n.º 24
0
def reset_is_running_on_all_spider( coll_model ) :
	"""
	reset is_running on all spiders to avoid errors if app shut down while one spider was running 
	"""

	print ()

	app_log.warning('>>> reset_is_running_on_all_spider ... ')
	
	# find if any spider was running
	running_spiders = coll_model.find({"scraper_log.is_running" : True})
	app_log.info(">>> running_spiders : \n %s" , list(running_spiders) )

	coll_model.update_many({'scraper_log.is_running' : True }, {"$set": {'scraper_log.is_running' : False }})
Ejemplo n.º 25
0
    def check_hub_user(self, model):
        """Check whether Hub-authenticated user or service should be allowed.

        Returns the input if the user should be allowed, None otherwise.

        Override if you want to check anything other than the username's presence in hub_users list.

        Args:
            model (dict): the user or service model returned from :class:`HubAuth`
        Returns:
            user_model (dict): The user model if the user should be allowed, None otherwise.
        """

        name = model['name']
        kind = model.setdefault('kind', 'user')
        if self.allow_all:
            app_log.debug(
                "Allowing Hub %s %s (all Hub users and services allowed)", kind, name
            )
            return model

        if self.allow_admin and model.get('admin', False):
            app_log.debug("Allowing Hub admin %s", name)
            return model

        if kind == 'service':
            # it's a service, check hub_services
            if self.hub_services and name in self.hub_services:
                app_log.debug("Allowing whitelisted Hub service %s", name)
                return model
            else:
                app_log.warning("Not allowing Hub service %s", name)
                raise UserNotAllowed(model)

        if self.hub_users and name in self.hub_users:
            # user in whitelist
            app_log.debug("Allowing whitelisted Hub user %s", name)
            return model
        elif self.hub_groups and set(model['groups']).intersection(self.hub_groups):
            allowed_groups = set(model['groups']).intersection(self.hub_groups)
            app_log.debug(
                "Allowing Hub user %s in group(s) %s",
                name,
                ','.join(sorted(allowed_groups)),
            )
            # group in whitelist
            return model
        else:
            app_log.warning("Not allowing Hub user %s", name)
            raise UserNotAllowed(model)
Ejemplo n.º 26
0
def backup_mongo_collection(coll, filepath) :
	"""
	dumps all documents in collection in _backups_collections 
	"""

	app_log.warning('>>> backup_mongo_collection ... ')

	cursor 		= coll.find({})
	backup_file = open(filepath, "w")
	backup_file.write('[')
	for document in cursor:
		backup_file.write(json.dumps(document,indent=4, default=json_util.default))
		backup_file.write(',')
	backup_file.write(']')
Ejemplo n.º 27
0
    def observe(self):
        '''Collect Ground Truth of what's actually running from Docker and the proxy.'''

        results = yield {
            "docker": self.spawner.list_notebook_servers(self.container_config, all=True),
            "proxy": self._proxy_routes()
        }

        self.container_ids = set()
        self.living_container_ids = []
        self.stopped_container_ids = []
        self.zombie_container_ids = []

        self.routes = set()
        self.live_routes = []
        self.stale_routes = []
        self.zombie_routes = []

        # Sort Docker results into living and dead containers.
        for container in results["docker"]:
            id = container['Id']
            self.container_ids.add(id)
            if container['Status'].startswith('Up'):
                self.living_container_ids.append(id)
            else:
                self.stopped_container_ids.append(id)

        cutoff = datetime.utcnow() - self.cull_time

        # Sort proxy routes into living, stale, and zombie routes.
        living_set = set(self.living_container_ids)
        for path, route in results["proxy"].items():
            last_activity_s = route.get('last_activity', None)
            container_id = route.get('container_id', None)
            if container_id:
                result = (path, container_id)
                if container_id in living_set:
                    try:
                        last_activity = datetime.strptime(last_activity_s, '%Y-%m-%dT%H:%M:%S.%fZ')

                        self.routes.add(result)
                        if last_activity >= cutoff:
                            self.live_routes.append(result)
                        else:
                            self.stale_routes.append(result)
                    except ValueError as e:
                        app_log.warning("Ignoring a proxy route with an unparsable activity date: %s", e)
                else:
                    # The container doesn't correspond to a living container.
                    self.zombie_routes.append(result)
Ejemplo n.º 28
0
 def post(self):
     '''Spawns a brand new server programmatically'''
     try:
         container = self.pool.acquire()
         url = container.path
         if container.token:
             url = url_concat(url, {'token': container.token})
         app_log.info("Allocated [%s] from the pool.", url)
         app_log.debug("Responding with container url [%s].", url)
         self.write({'url': url})
     except spawnpool.EmptyPoolError:
         app_log.warning("The container pool is empty!")
         self.set_status(429)
         self.write({'status': 'full'})
Ejemplo n.º 29
0
    def user_for_cookie(self, encrypted_cookie, use_cache=True):
        """Ask the Hub to identify the user for a given cookie.

        Args:
            encrypted_cookie (str): the cookie value (not decrypted, the Hub will do that)
            use_cache (bool): Specify use_cache=False to skip cached cookie values (default: True)

        Returns:
            user_model (dict): The user model, if a user is identified, None if authentication fails.

            The 'name' field contains the user's name.
        """
        if use_cache:
            cached = self.cookie_cache.get(encrypted_cookie)
            if cached is not None:
                return cached
        try:
            r = requests.get(
                url_path_join(self.api_url,
                              "authorizations/cookie",
                              self.cookie_name,
                              quote(encrypted_cookie, safe=''),
                ),
                headers = {
                    'Authorization' : 'token %s' % self.api_token,
                },
            )
        except requests.ConnectionError:
            msg = "Failed to connect to Hub API at %r." % self.api_url
            msg += "  Is the Hub accessible at this URL (from host: %s)?" % socket.gethostname()
            if '127.0.0.1' in self.api_url:
                msg += "  Make sure to set c.JupyterHub.hub_ip to an IP accessible to" + \
                       " single-user servers if the servers are not on the same host as the Hub."
            raise HTTPError(500, msg)

        if r.status_code == 404:
            data = None
        elif r.status_code == 403:
            app_log.error("I don't have permission to verify cookies, my auth token may have expired: [%i] %s", r.status_code, r.reason)
            raise HTTPError(500, "Permission failure checking authorization, I may need a new token")
        elif r.status_code >= 500:
            app_log.error("Upstream failure verifying auth token: [%i] %s", r.status_code, r.reason)
            raise HTTPError(502, "Failed to check authorization (upstream problem)")
        elif r.status_code >= 400:
            app_log.warning("Failed to check authorization: [%i] %s", r.status_code, r.reason)
            raise HTTPError(500, "Failed to check authorization")
        else:
            data = r.json()
        self.cookie_cache[encrypted_cookie] = data
        return data
Ejemplo n.º 30
0
    def __run(self, ws):
        """Receives message from a WebSocket, sends them to the client's
        listeners.
        """
        while True:
            msg_str = yield ws.read_message()
            if msg_str is None:
                break
            try:
                msg_json = json.loads(msg_str)
            except (TypeError, ValueError):
                log.error('Invalid event: {0}'.format(msg_str))
                continue
            if not isinstance(msg_json, dict) or 'type' not in msg_json:
                log.error('Invalid event: {0}'.format(msg_str))
                continue

            log.debug('Process ARI event: {0}'.format(msg_json))
            event_type = msg_json['type']
            listeners = self.event_listeners.get(event_type)
            if listeners:
                log.debug('Listeners of event type "{0}": {1}'
                          .format(event_type, len(listeners)))
                # Extract objects from the event
                event_model = self.event_models.get(event_type)
                if not event_model:
                    log.warning('Cannot find model "{0}" for received event. '
                                'Pass raw event.'.format(event_type))
                    event = msg_json
                else:
                    event = dict()
                    properties = event_model['properties']
                    for field, value in msg_json.items():
                        if field in properties:
                            type_ = properties[field]['type']
                            if type_ in CLASS_MAP:
                                value = CLASS_MAP[type_](self, value)
                        event[field] = value

                # Set a result of pending futures
                for listener in listeners:
                    future, event_filter = listener
                    if future.done():
                        listeners.remove(listener)
                        continue

                    if event_filter is None or event_filter(event):
                            future.set_result(event)
                            listeners.remove(listener)
Ejemplo n.º 31
0
    def get_json_body(self, allow_empty=False):
        """Retrieve and validate a JSON request body"""
        try:
            return self._json_body
        except AttributeError:
            pass

        if allow_empty and not self.request.body:
            body = self._json_body = {}
            return body

        try:
            body = self._json_body = json.loads(
                self.request.body.decode("utf8", "replace"))
            if not isinstance(body, dict):
                raise ValueError("json body is not a dict")
        except ValueError:
            raise web.HTTPError(
                400,
                f"Request body must be a json dict of the form {make_sample_schema(self.schema)}",
            )

        schema = self.schema

        for field, field_type in schema.items():
            if isinstance(field_type, tuple):
                field_type, required = field_type
            else:
                required = True
            value = body.get(field)
            if value and field_type is datetime.datetime:
                try:
                    value = body[field] = parse_date(body[field])
                except Exception as e:
                    app_log.warning(f"Error parsing date: {body[field]}")

            if (field in body and not isinstance(value, field_type)) or (
                    required and field not in body):
                sample_schema = make_sample_schema(schema)
                raise web.HTTPError(
                    400,
                    f"Field {field} must be {type_map.get(field_type, field_type.__name__)}. A request body should look like: {sample_schema}",
                )

        return body
Ejemplo n.º 32
0
    def _cleanup(cls):
        def timeout_waiter(socket, timeout):
            flags = []

            def _ping_waiter(data):
                flags.append(True)
                log.debug('Socket "%r" ping OK', socket)

            def _timeout():
                if not flags:
                    try:
                        log.warning('Socket "%r" must be closed', socket)
                        socket.close()
                    except Exception as e:
                        log.debug(Lazy(lambda: traceback.format_exc()))
                        log.error("%r", e)
                if hasattr(socket, 'on_pong'):
                    delattr(socket, 'on_pong')

            tornado.ioloop.IOLoop.instance().call_later(timeout, _timeout)
            return _ping_waiter

        def do_ping(socket):
            if isinstance(socket.ws_connection,
                          tornado.websocket.WebSocketProtocol13):
                socket.ping("\0" * 8)
                socket.on_pong = timeout_waiter(socket, cls._CLIENT_TIMEOUT)
            else:
                socket.call('ping',
                            data="ping",
                            callback=timeout_waiter(socket,
                                                    cls._CLIENT_TIMEOUT))
                sleep(cls._CLIENT_TIMEOUT)

        for sid, socket in cls._CLIENTS.iteritems():
            try:
                do_ping(socket)
            except tornado.websocket.WebSocketClosedError:
                socket.close()
                log.warning('Auto close dead socket: %r', socket)
            except Exception as e:
                log.debug(Lazy(lambda: traceback.format_exc()))
                log.error('%r', e)

        log.debug('Cleanup loop is OK.')
Ejemplo n.º 33
0
    def fetch_impl(self, request, response_callback):
        urlinfo = urlparse(request.url)
        host = urlinfo.hostname
        if host not in self.hosts:
            app_log.warning("Not mocking request to %s", request.url)
            return super().fetch_impl(request, response_callback)
        paths = self.hosts[host]
        response = None
        for path_spec, handler in paths:
            if isinstance(path_spec, str):
                if path_spec == urlinfo.path:
                    response = handler(request)
                    break
            else:
                if path_spec.match(urlinfo.path):
                    response = handler(request)
                    break

        if response is None:
            response = HTTPResponse(request=request,
                                    code=404,
                                    reason=request.url)
        elif isinstance(response, int):
            response = HTTPResponse(request=request, code=response)
        elif isinstance(response, bytes):
            response = HTTPResponse(
                request=request,
                code=200,
                buffer=BytesIO(response),
            )
        elif isinstance(response, str):
            response = HTTPResponse(
                request=request,
                code=200,
                buffer=BytesIO(response.encode('utf8')),
            )
        elif isinstance(response, (dict, list)):
            response = HTTPResponse(
                request=request,
                code=200,
                buffer=BytesIO(json.dumps(response).encode('utf8')),
                headers={'Content-Type': 'application/json'},
            )

        response_callback(response)
Ejemplo n.º 34
0
async def get_group(device_id, select=None):
    """Get a single device group"""
    if select is None:
        attr_names = ",".join(
            extension_attr_name(attr["name"]) for attr in custom_attributes
            if "Group" in attr["targetObjects"])
        select = f"id,displayName,createdDateTime,{attr_names}"
    groups = await graph_request(
        "/groups",
        params={
            "$filter": f"displayName eq '{device_id}'",
            "$select": select,
        },
    )
    if not groups:
        app_log.warning(f"No group for device {device_id}")
        return
    return groups[0]
Ejemplo n.º 35
0
 def _readNotebooks(self):
     app_log.debug("Reading notebooks from notebook_dir %s", self.notebook_dir)
     if self.notebook_dir is None:
         app_log.warning("No notebooks to load")
         return
     for path in os.listdir(self.notebook_dir):
         if path.endswith(".ipynb"):
             full_path = os.path.join(self.notebook_dir, path)
             nb_contents = self.loader.load(full_path)
             if nb_contents is not None:
                 with nb_contents:
                     app_log.debug("loading Notebook: %s",path)
                     notebook = nbformat.read(nb_contents, as_version=4)
                     #Load the pixieapp definition if any
                     pixieapp_def = self.read_pixieapp_def(notebook)
                     if pixieapp_def is not None and pixieapp_def.is_valid:
                         pixieapp_def.location = full_path
                         self.pixieapps[pixieapp_def.name] = pixieapp_def
Ejemplo n.º 36
0
 def check_request_ip(self):
     """Check network block list, if any"""
     ban_networks = self.settings.get("ban_networks")
     if self.skip_check_request_ip or not ban_networks:
         return
     request_ip = self.request.remote_ip
     match = ip_in_networks(
         request_ip,
         ban_networks,
         min_prefix_len=self.settings["ban_networks_min_prefix_len"],
     )
     if match:
         network, message = match
         app_log.warning(
             f"Blocking request from {request_ip} matching banned network {network}: {message}"
         )
         raise web.HTTPError(403,
                             f"Requests from {message} are not allowed")
    def on_connection_closed(self, connection, reply_code, reply_text):
        """This method is invoked by pika when the connection to RabbitMQ is
        closed unexpectedly. Since it is unexpected, we will reconnect to
        RabbitMQ if it disconnects.

        :param pika.connection.Connection connection: The closed connection obj
        :param int reply_code: The server provided reply_code if given
        :param str reply_text: The server provided reply_text if given

        """
        self._channel = None
        if self._closing:
            self._connection.ioloop.stop()
        else:
            app_log.warning(
                'Connection closed, reopening in 5 seconds: (%s) %s',
                reply_code, reply_text)
            self._connection.add_timeout(5, self.reconnect)
Ejemplo n.º 38
0
 def send_mail(self, subject='[系统邮件]', message='', to=list()):
     smtp_client, res = yield self._smtp_client()
     if smtp_client is not None:
         msg = MIMEText(message, 'html', 'utf-8')
         msg['Subject'] = subject
         msg['Sender'] = self.smtp_user
         msg['To'] = ', '.join(to)
         msg['From'] = formataddr(parseaddr('自动化测试系统邮件<{}>'.format(self.mail_from)))
         try:
             yield smtp_client.send_message(msg)
             yield smtp_client.quit()
             return True, ''
         except Exception as e:
             log.warning(e)
             yield smtp_client.quit()
             return False, e
     else:
         return False, res
Ejemplo n.º 39
0
 def on_reply(msg):
     if 'msg_id' in msg['parent_header'] and msg['parent_header']['msg_id'] == parent_header:
         if not future.done():
             if "channel" not in msg:
                 msg["channel"] = "iopub"
             result_accumulator.append(msg)
             # Complete the future on idle status
             if msg['header']['msg_type'] == 'status' and msg['content']['execution_state'] == 'idle':
                 future.set_result(result_extractor(result_accumulator))
             elif msg['header']['msg_type'] == 'error':
                 error_name = msg['content']['ename']
                 error_value = msg['content']['evalue']
                 trace = sanitize_traceback(msg['content']['traceback'])
                 future.set_exception(
                     CodeExecutionError(error_name, error_value, trace, code)
                 )
     else:
         app_log.warning("Got an orphan message %s", msg['parent_header'])
Ejemplo n.º 40
0
 def result_extractor(self, result_accumulator):
     res = []
     for msg in result_accumulator:
         if msg['header']['msg_type'] == 'stream':
             res.append(msg['content']['text'])
         elif msg['header']['msg_type'] == 'display_data':
             if "data" in msg['content'] and "text/html" in msg['content']['data']:
                 res.append(msg['content']['data']['text/html'])
             else:
                 app_log.warning("display_data msg not processed: %s", msg)                    
         elif msg['header']['msg_type'] == 'error':
             error_name = msg['content']['ename']
             error_value = msg['content']['evalue']
             trace = sanitize_traceback(msg['content']['traceback'])
             return 'Error {}: {}\n{}\n'.format(error_name, error_value, trace)
         else:
             app_log.warning("Message type not processed: %s", msg['header']['msg_type'])
     return ''.join(res)
Ejemplo n.º 41
0
    def parse_remote(self, response):
        if self._finished:
            app_log.info('connection canceled')
            return

        try:
            if response.code != 200:
                app_log.warning(
                    'Error while getting remote %s '
                    'Errors details: (%s: %s) %s', response.effective_url,
                    response.code, response.reason, response.body)
                return

            base_url = response.effective_url
            base = urlsplit(base_url)

            content_type = response.headers.get('content-type', '')
            if content_type in ('application/x-gzip', ):
                # in this case the URL was a redirection to download
                # a package. For example, sourceforge.
                self.add_version(basename(base.path), '', base_url, base_url)
                return

            if not response.body:
                return

            app_log.debug('parse %s', base_url)

            soup = BeautifulSoup(response.body)
            for anchor in soup.find_all('a'):
                href = anchor.get('href')
                if not href:
                    continue

                current_url = urljoin(base_url, href)
                current = urlsplit(current_url)

                if self.is_archive(current.path):
                    self.add_version(basename(current.path), '', current_url,
                                     href)
                else:
                    self.add_link(current_url, current, base, href)
        finally:
            self.fetch_next()
Ejemplo n.º 42
0
    def set_state_cookie(self, handler, next_url=None):
        """Generate an OAuth state and store it in a cookie

        Parameters
        ----------
        handler : RequestHandler
            A tornado RequestHandler
        next_url : str
            The page to redirect to on successful login

        Returns
        -------
        state : str
            The OAuth state that has been stored in the cookie (url safe, base64-encoded)
        """
        extra_state = {}
        if handler.get_cookie(self.state_cookie_name):
            # oauth state cookie is already set
            # use a randomized cookie suffix to avoid collisions
            # in case of concurrent logins
            app_log.warning("Detected unused OAuth state cookies")
            cookie_suffix = ''.join(
                random.choice(string.ascii_letters) for i in range(8)
            )
            cookie_name = '{}-{}'.format(self.state_cookie_name, cookie_suffix)
            extra_state['cookie_name'] = cookie_name
        else:
            cookie_name = self.state_cookie_name
        b64_state = self.generate_state(next_url, **extra_state)
        kwargs = {
            'path': self.base_url,
            'httponly': True,
            # Expire oauth state cookie in ten minutes.
            # Usually this will be cleared by completed login
            # in less than a few seconds.
            # OAuth that doesn't complete shouldn't linger too long.
            'max_age': 600,
        }
        if handler.request.protocol == 'https':
            kwargs['secure'] = True
        # load user cookie overrides
        kwargs.update(self.cookie_options)
        handler.set_secure_cookie(cookie_name, b64_state, **kwargs)
        return b64_state
Ejemplo n.º 43
0
    def _api_request(self, method, url, **kwargs):
        """Make an API request"""
        allow_404 = kwargs.pop('allow_404', False)
        headers = kwargs.setdefault('headers', {})
        headers.setdefault('Authorization', 'token %s' % self.api_token)
        try:
            r = requests.request(method, url, **kwargs)
        except requests.ConnectionError as e:
            app_log.error("Error connecting to %s: %s", self.api_url, e)
            msg = "Failed to connect to Hub API at %r." % self.api_url
            msg += "  Is the Hub accessible at this URL (from host: %s)?" % socket.gethostname(
            )
            if '127.0.0.1' in self.api_url:
                msg += "  Make sure to set c.JupyterHub.hub_ip to an IP accessible to" + \
                       " single-user servers if the servers are not on the same host as the Hub."
            raise HTTPError(500, msg)

        data = None
        if r.status_code == 404 and allow_404:
            pass
        elif r.status_code == 403:
            app_log.error(
                "I don't have permission to check authorization with JupyterHub, my auth token may have expired: [%i] %s",
                r.status_code, r.reason)
            app_log.error(r.text)
            raise HTTPError(
                500,
                "Permission failure checking authorization, I may need a new token"
            )
        elif r.status_code >= 500:
            app_log.error("Upstream failure verifying auth token: [%i] %s",
                          r.status_code, r.reason)
            app_log.error(r.text)
            raise HTTPError(
                502, "Failed to check authorization (upstream problem)")
        elif r.status_code >= 400:
            app_log.warning("Failed to check authorization: [%i] %s",
                            r.status_code, r.reason)
            app_log.warning(r.text)
            raise HTTPError(500, "Failed to check authorization")
        else:
            data = r.json()

        return data
Ejemplo n.º 44
0
    def check_hub_user(self, model):
        """Check whether Hub-authenticated user or service should be allowed.

        Returns the input if the user should be allowed, None otherwise.

        Override if you want to check anything other than the username's presence in hub_users list.

        Args:
            model (dict): the user or service model returned from :class:`HubAuth`
        Returns:
            user_model (dict): The user model if the user should be allowed, None otherwise.
        """

        name = model['name']
        kind = model.setdefault('kind', 'user')
        if self.allow_all:
            app_log.debug("Allowing Hub %s %s (all Hub users and services allowed)", kind, name)
            return model

        if self.allow_admin and model.get('admin', False):
            app_log.debug("Allowing Hub admin %s", name)
            return model

        if kind == 'service':
            # it's a service, check hub_services
            if self.hub_services and name in self.hub_services:
                app_log.debug("Allowing whitelisted Hub service %s", name)
                return model
            else:
                app_log.warning("Not allowing Hub service %s", name)
                raise UserNotAllowed(model)

        if self.hub_users and name in self.hub_users:
            # user in whitelist
            app_log.debug("Allowing whitelisted Hub user %s", name)
            return model
        elif self.hub_groups and set(model['groups']).intersection(self.hub_groups):
            allowed_groups = set(model['groups']).intersection(self.hub_groups)
            app_log.debug("Allowing Hub user %s in group(s) %s", name, ','.join(sorted(allowed_groups)))
            # group in whitelist
            return model
        else:
            app_log.warning("Not allowing Hub user %s", name)
            raise UserNotAllowed(model)
Ejemplo n.º 45
0
 async def proxy(self):
     client = AsyncHTTPClient()
     try:
         app_log.info('Proxying {} {}'.format(self.request.method, self.request.path))
         resp = await client.fetch(urljoin(NPM_REGISTRY_URL, self.request.path),
                                   method=self.request.method,
                                   body=self.request.body if self.request.method in ['PUT', 'POST'] else None,
                                   headers={name: value for name, value in self.request.headers.get_all() if name.lower() not in ['host', 'connection', 'user-agent']},
                                   follow_redirects=False, raise_error=True)
     except HTTPClientError as err:
         self.set_status(err.code)
         self.finish(err.response.body)
         return
     except Exception as err:
         app_log.warning('Error connecting to npm', exc_info=sys.exc_info)
         self.send_error(503)
         return
     # TODO: stream instead of bufferring
     self.finish(resp.body)
Ejemplo n.º 46
0
    def access_token(request):
        """Handler for access token endpoint

        Checks code and allocates a new token.
        Replies with JSON model for the token.
        """
        assert request.method == 'POST'
        if token_request_style == 'json':
            body = request.body.decode('utf8')
            try:
                body = json.loads(body)
            except ValueError:
                return HTTPResponse(request=request, code=400,
                    reason="Body not JSON: %r" % body,
                )
            else:
                code = body['code']
        else:
            query = urlparse(request.url).query
            if not query:
                query = request.body.decode('utf8')
            query = parse_qs(query)
            if 'code' not in query:
                return HTTPResponse(request=request, code=400,
                    reason="No code in access token request: url=%s, body=%s" % (
                        request.url, request.body,
                    )
                )
            code = query['code'][0]
        if code not in oauth_codes:
            app_log.warning()
            return HTTPResponse(request=request, code=403,
                reason="No such code: %s" % code,
            )

        # consume code, allocate token
        token = uuid.uuid4().hex
        user = oauth_codes.pop(code)
        access_tokens[token] = user
        return {
            'access_token': token,
            'token_type': token_type,
        }
Ejemplo n.º 47
0
 async def is_reachable():
     try:
         r = await client.fetch(url, follow_redirects=False)
         return r
     except HTTPError as e:
         if e.code >= 500:
             # failed to respond properly, wait and try again
             if e.code != 599:
                 # we expect 599 for no connection,
                 # but 502 or other proxy error is conceivable
                 app_log.warning(
                     "Server at %s responded with error: %s", url, e.code)
         else:
             app_log.debug("Server at %s responded with %s", url, e.code)
             return e.response
     except (OSError, socket.error) as e:
         if e.errno not in {errno.ECONNABORTED, errno.ECONNREFUSED, errno.ECONNRESET}:
             app_log.warning("Failed to connect to %s (%s)", url, e)
     return False
Ejemplo n.º 48
0
 def authenticate_client_id(self, client_id, request, *args, **kwargs):
     """Ensure client_id belong to a non-confidential client.
     A non-confidential client is one that is not required to authenticate
     through other means, such as using HTTP Basic.
     Note, while not strictly necessary it can often be very convenient
     to set request.client to the client object associated with the
     given client_id.
     :param request: oauthlib.common.Request
     :rtype: True or False
     Method is used by:
         - Authorization Code Grant
     """
     orm_client = (self.db.query(
         orm.OAuthClient).filter_by(identifier=client_id).first())
     if orm_client is None:
         app_log.warning("No such oauth client %s", client_id)
         return False
     request.client = orm_client
     return True
Ejemplo n.º 49
0
    def prepare(self):
        # copy hosts config in case it changes while we are iterating over it
        hosts = dict(self.settings["hosts"])
        if not hosts:
            # no healthy hosts, allow routing to unhealthy 'prime' host only
            hosts = {
                key: host
                for key, host in CONFIG["hosts"].items() if host.get("prime")
            }
            app_log.warning(
                f"Using unhealthy prime host(s) {list(hosts)} because zero hosts are healthy"
            )
        self.host_names = [c["url"] for c in hosts.values() if c["weight"] > 0]
        self.host_weights = [
            c["weight"] for c in hosts.values() if c["weight"] > 0
        ]

        # Combine hostnames and weights into one list
        self.names_and_weights = list(zip(self.host_names, self.host_weights))
Ejemplo n.º 50
0
    def add_error_message_to_slug(self,
                                  error_string,
                                  args_to_delete=[],
                                  msg_categ="error"):
        """ add an "error" arg to url slug """

        slug_ = self.request.arguments
        app_log.info("... add_error_message_to_slug / slug_ : \n %s ",
                     pformat(slug_))

        # create a complete clean slug if no slug
        if slug_ == {}:
            error_slug = u"?" + u"error=" + tornado.escape.url_escape(
                error_string)

        # add error arg to existing slug
        else:

            slug_without_error = deepcopy(slug_)

            ### clean existing slug from existing error arg if any
            # for arg_to_delete in args_to_delete + DEFAULT_ERROR_ARGS_TO_DELETE :
            # 	try :
            # 		del slug_without_error[arg_to_delete]
            # 	except :
            # 		pass
            slug_without_error = self.clean_slug(
                slug_without_error,
                args_to_delete + DEFAULT_ERROR_ARGS_TO_DELETE)

            app_log.warning(
                "... add_error_message_to_slug / slug_without_error : \n %s ",
                pformat(slug_without_error))

            # recreate slug
            error_dict = {"error": error_string}
            error_dict.update(slug_without_error)
            error_slug = u"?" + urllib.urlencode(error_dict, doseq=True)

        app_log.info("... add_error_message_to_slug / error_slug : \n %s ",
                     pformat(error_slug))

        return error_slug
Ejemplo n.º 51
0
 def get(self, op='reports', page=1, limit=10):
     if not isinstance(limit, int):
         limit = int(limit)
     else:
         limit = limit if self.limit == '' else int(self.limit)
     try:
         page = int(page)
     except Exception as e:
         log.warning(e)
         page = 1
     page = 1 if int(page) <= 0 else int(page)
     lists = []
     total_page = 1
     if op == 'reports':
         res, total = yield self.setting.get_settings_list(
             s_type='page_report')
         total_page = int(math.ceil(total / limit))
         for row in res:
             report = json.loads(row.value)
             report['sid'] = row.id
             lists.append(munchify(report))
     elif op not in ['checklinks', 'checkpages']:
         self.redirect('/admin/page-test')
         return
     hosts = ''
     if platform.system().lower() == 'Windows'.lower():
         host_path = 'C:\\Windows\\System32\\drivers\\etc\\hosts'
     else:
         host_path = '/etc/hosts'
     if os.path.exists(host_path):
         with open(host_path, 'r', encoding='utf8') as fp:
             hosts = fp.read()
             log.info('读取Hosts {} 配置成功'.format(host_path))
     argv = dict(title='页面监控',
                 op=op,
                 lists=lists,
                 hosts=hosts,
                 total_page=total_page,
                 page=page,
                 limit=limit)
     argv = dict(self.argv, **argv)
     self.render('admin/page.html', **argv)
Ejemplo n.º 52
0
    def check_origin(self, origin_to_satisfy_tornado=""):
        """Check Origin for cross-site API requests, including websockets

        Copied from WebSocket with changes:

        - allow unspecified host/origin (e.g. scripts)
        - allow token-authenticated requests
        """
        if self.allow_origin == '*':
            return True

        host = self.request.headers.get("Host")
        origin = self.request.headers.get("Origin")

        # If no header is provided, assume it comes from a script/curl.
        # We are only concerned with cross-site browser stuff here.
        if origin is None or host is None:
            return True

        origin = origin.lower()
        origin_host = urlparse(origin).netloc

        # OK if origin matches host
        if origin_host == host:
            return True

        # Check CORS headers
        if self.allow_origin:
            allow = self.allow_origin == origin
        elif self.allow_origin_pat:
            allow = bool(self.allow_origin_pat.match(origin))
        else:
            # No CORS headers deny the request
            allow = False
        if not allow:
            app_log.warning(
                'Blocking Cross Origin API request for %s.  Origin: %s, Host: %s',
                self.request.path,
                origin,
                host,
            )
        return allow
Ejemplo n.º 53
0
    def phantom_parsing_call(self, url):
        app_log.info('phantom process start %s' % url)
        # todo proxy servers
        # todo logs path
        js_parser_path = os.path.sep.join([self.application.app_path, 'js_parser'])
        phantom_parser_cmd = "%s --ignore-ssl-errors='true' --load-images='false' %s %swebfont_preview.html"\
                             % (os.path.sep.join([js_parser_path, 'phantomjs-1.9.8', 'phantomjs']),
                                os.path.sep.join([js_parser_path, 'parser2.js']), url)
        app_log.info('command: <<%s>>' % phantom_parser_cmd)

        try:
            result, error = yield self.call_subprocess(phantom_parser_cmd)
        except Exception as e:
            app_log.warning('phantom crash: <<%s>>' % e)
            raise gen.Return(False)

        app_log.info('phantom process end: <<%d>> <<%d>>' % (len(result), len(error)))
        self.phantom_logs_save(url, result, error)

        raise gen.Return(self.phantom_result_parse(result))
Ejemplo n.º 54
0
 def token(self, token):
     """Store the hashed value and prefix for a token"""
     self.prefix = token[:self.prefix_length]
     if len(token) >= 32:
         # Tokens are generally UUIDs, which have sufficient entropy on their own
         # and don't need salt & hash rounds.
         # ref: https://security.stackexchange.com/a/151262/155114
         rounds = 1
         salt_bytes = b''
     else:
         # users can still specify API tokens in a few ways,
         # so trigger salt & hash rounds if they provide a short token
         app_log.warning("Applying salt & hash rounds to %sB token" %
                         len(token))
         rounds = self.rounds
         salt_bytes = self.salt_bytes
     self.hashed = hash_token(token,
                              rounds=rounds,
                              salt=salt_bytes,
                              algorithm=self.algorithm)
Ejemplo n.º 55
0
    def __init__(self, **kwargs):

        print
        app_log.warning("== UserClass ... ")

        app_log.warning("== UserClass / fields from USER_CORE_FIELDS ... ")
        for user_field in USER_CORE_FIELDS:
            app_log.warning("== UserClass / user_field : %s", user_field)
            self.__dict__[user_field] = ""

        app_log.warning("== UserClass / fields from **kwargs ...")
        for k, v in kwargs.items():
            # print "{} : {}".format(k,v)
            app_log.warning("== UserClass / %s : %s ...", k, v)
            try:
                self.__dict__[k] = v
            except:
                pass

        print
Ejemplo n.º 56
0
def sig_handler(server, shutdown_waittime, sig, frame):
    io_loop = tornado.ioloop.IOLoop.instance()

    def stop_loop(deadline):
        now = time.time()
        if now < deadline and (io_loop._callbacks or io_loop._timeouts):
            app_log.info('Waiting for next tick')
            io_loop.add_timeout(now + 1, stop_loop, deadline)
        else:
            io_loop.stop()
            app_log.info('Shutdown finally')

    def shutdown():
        app_log.info('Stopping http server')
        server.stop()
        app_log.info('Will shutdown in %s seconds ...', shutdown_waittime)
        stop_loop(time.time() + shutdown_waittime)

    app_log.warning('Caught signal: %s', sig)
    io_loop.add_callback_from_signal(shutdown)
Ejemplo n.º 57
0
    def _check_hub_authorization(self,
                                 url,
                                 api_token,
                                 cache_key=None,
                                 use_cache=True):
        """Identify a user with the Hub

        Args:
            url (str): The API URL to check the Hub for authorization
                       (e.g. http://127.0.0.1:8081/hub/api/user)
            cache_key (str): The key for checking the cache
            use_cache (bool): Specify use_cache=False to skip cached cookie values (default: True)

        Returns:
            user_model (dict): The user model, if a user is identified, None if authentication fails.

        Raises an HTTPError if the request failed for a reason other than no such user.
        """
        if use_cache:
            if cache_key is None:
                raise ValueError("cache_key is required when using cache")
            # check for a cached reply, so we don't check with the Hub if we don't have to
            try:
                return self.cache[cache_key]
            except KeyError:
                app_log.debug("HubAuth cache miss: %s", cache_key)

        data = self._api_request(
            'GET',
            url,
            headers={"Authorization": "token " + api_token},
            allow_403=True,
        )
        if data is None:
            app_log.warning("No Hub user identified for request")
        else:
            app_log.debug("Received request from Hub user %s", data)
        if use_cache:
            # cache result
            self.cache[cache_key] = data
        return data
Ejemplo n.º 58
0
    def post(self):
        uri, http_method, body, headers = self.extract_oauth_params()
        referer = self.request.headers.get('Referer', 'no referer')
        full_url = self.request.full_url()
        # trim protocol, which cannot be trusted with multiple layers of proxies anyway
        # Referer is set by browser, but full_url can be modified by proxy layers to appear as http
        # when it is actually https
        referer_proto, _, stripped_referer = referer.partition("://")
        referer_proto = referer_proto.lower()
        req_proto, _, stripped_full_url = full_url.partition("://")
        req_proto = req_proto.lower()
        if referer_proto != req_proto:
            app_log.warning("Protocol mismatch: %s != %s", referer, full_url)
            if req_proto == "https":
                # insecure origin to secure target is not allowed
                raise web.HTTPError(
                    403,
                    "Not allowing authorization form submitted from insecure page"
                )
        if stripped_referer != stripped_full_url:
            # OAuth post must be made to the URL it came from
            app_log.error("Original OAuth POST from %s != %s", referer,
                          full_url)
            app_log.error("Stripped OAuth POST from %s != %s",
                          stripped_referer, stripped_full_url)
            raise web.HTTPError(
                403, "Authorization form must be sent from authorization page")

        # The scopes the user actually authorized, i.e. checkboxes
        # that were selected.
        scopes = self.get_arguments('scopes')
        # credentials we need in the validator
        credentials = self.add_credentials()

        try:
            headers, body, status = self.oauth_provider.create_authorization_response(
                uri, http_method, body, headers, scopes, credentials)
        except oauth2.FatalClientError as e:
            raise web.HTTPError(e.status_code, e.description)
        else:
            self.send_oauth_response(headers, body, status)
Ejemplo n.º 59
0
async def find_groups_to_delete():
    filter = f"{to_delete} eq true"
    if BACKLOG_DATE:
        filter = f"{filter} and {to_delete_date} le {BACKLOG_DATE}"
    count = 0
    async for group in graph.list_groups(filter=filter):
        if not group.get(to_delete):
            raise RuntimeError(
                f"Group {group['displayName']} does not have toDelete set!"
            )
        if not group.get(to_delete_date):
            app_log.warning(
                f"Group {group['displayName']} marked toDelete, but no date! Saving for later."
            )
            await graph.mark_for_deletion(group)
            continue
        count += 1
        if count % 100 == 0:
            app_log.info(f"Found {count} devices to delete")
        yield group
    app_log.info(f"Found {count} total devices to delete")
Ejemplo n.º 60
0
def get_browser_protocol(request):
    """Get the _protocol_ seen by the browser

    Like tornado's _apply_xheaders,
    but in the case of multiple proxy hops,
    use the outermost value (what the browser likely sees)
    instead of the innermost value,
    which is the most trustworthy.

    We care about what the browser sees,
    not where the request actually came from,
    so trusting possible spoofs is the right thing to do.
    """
    headers = request.headers
    # first choice: Forwarded header
    forwarded_header = headers.get("Forwarded")
    if forwarded_header:
        first_forwarded = forwarded_header.split(",", 1)[0].strip()
        fields = {}
        forwarded_dict = {}
        for field in first_forwarded.split(";"):
            key, _, value = field.partition("=")
            fields[key.strip().lower()] = value.strip()
        if "proto" in fields and fields["proto"].lower() in {"http", "https"}:
            return fields["proto"].lower()
        else:
            app_log.warning(
                f"Forwarded header present without protocol: {forwarded_header}"
            )

    # second choice: X-Scheme or X-Forwarded-Proto
    proto_header = headers.get("X-Scheme",
                               headers.get("X-Forwarded-Proto", None))
    if proto_header:
        proto_header = proto_header.split(",")[0].strip().lower()
        if proto_header in {"http", "https"}:
            return proto_header

    # no forwarded headers
    return request.protocol