def get_response(obj, service, url): problems = PerformanceDegradation.get_problems() if not problems: return delay = problems.get_load_time(service) if delay: time.sleep(float(delay)) status = problems.get_status(service) content = problems.get_content(service) if content and not status: status = 200 if status: response = MockHTTP() response.status = int(status) if content: response.data = content return response return None
def test_cache_expiration(self): with self.settings(RESTCLIENTS_DAO_CACHE_CLASS=CACHE): cache = BridgeAccountCache() ok_response = MockHTTP() ok_response.status = 200 ok_response.data = "xx" url = '/group/uw_member/effective_member' response = cache.getCache('gws', url, {}) self.assertEquals(response, None) cache.processResponse("gws", url, ok_response) response = cache.getCache("gws", url, {}) self.assertEquals(response["response"].data, 'xx') cache_entry = CacheEntryTimed.objects.get(service='gws', url=url) orig_time_saved = cache_entry.time_saved cache_entry.time_saved = (orig_time_saved - timedelta(minutes=(60 * 4) - 2)) cache_entry.save() response = cache.getCache("gws", url, {}) self.assertNotEquals(response, None) cache_entry.time_saved = (orig_time_saved - timedelta(minutes=(60 * 4) + 1)) cache_entry.save() response = cache.getCache("gws", url, {}) self.assertEquals(response, None)
def test_sws_default_policies(self): with self.settings(RESTCLIENTS_DAO_CACHE_CLASS=CACHE): cache = MyUWCache() ok_response = MockHTTP() ok_response.status = 200 ok_response.data = "xx" response = cache.getCache('sws', '/student/myuwcachetest1', {}) self.assertEquals(response, None) cache.processResponse("sws", "/student/myuwcachetest1", ok_response) response = cache.getCache('sws', '/student/myuwcachetest1', {}) self.assertEquals(response["response"].data, 'xx') cache_entry = CacheEntryTimed.objects.get( service="sws", url="/student/myuwcachetest1") # Cached response is returned after 3 hours and 58 minutes orig_time_saved = cache_entry.time_saved cache_entry.time_saved = (orig_time_saved - timedelta(minutes=(60 * 4) - 2)) cache_entry.save() response = cache.getCache('sws', '/student/myuwcachetest1', {}) self.assertNotEquals(response, None) # Cached response is not returned after 4 hours and 1 minute cache_entry.time_saved = (orig_time_saved - timedelta(minutes=(60 * 4) + 1)) cache_entry.save() response = cache.getCache('sws', '/student/myuwcachetest1', {}) self.assertEquals(response, None)
def _response_from_cache(self, service, url, headers, max_age_in_seconds, max_error_age=60 * 5): # If max_age_in_seconds is 0, # make sure we don't get a hit from this same second. if not max_age_in_seconds: return None now = make_aware(datetime.now(), get_current_timezone()) time_limit = now - timedelta(seconds=max_age_in_seconds) query = CacheEntryTimed.objects.filter(service=service, url=url, time_saved__gte=time_limit) if len(query): hit = query[0] if hit.status != 200 and (now - timedelta(seconds=max_error_age) > hit.time_saved): return None response = MockHTTP() response.status = hit.status response.data = hit.content response.headers = hit.getHeaders() return { "response": response, } return None
def test_default_policies(self): with self.settings(RESTCLIENTS_DAO_CACHE_CLASS=CACHE): cache = MyUWCache() ok_response = MockHTTP() ok_response.status = 200 ok_response.data = "xx" response = cache.getCache('no_such', '/student/myuwcachetest1', {}) self.assertEquals(response, None) cache.processResponse( "no_such", "/student/myuwcachetest1", ok_response) response = cache.getCache('no_such', '/student/myuwcachetest1', {}) self.assertEquals(response["response"].data, 'xx') cache_entry = CacheEntryTimed.objects.get( service="no_such", url="/student/myuwcachetest1") # Cached response is returned after 3 hours and 58 minutes orig_time_saved = cache_entry.time_saved cache_entry.time_saved = (orig_time_saved - timedelta(minutes=(60 * 4)-2)) cache_entry.save() response = cache.getCache('no_such', '/student/myuwcachetest1', {}) self.assertNotEquals(response, None) # Cached response is not returned after 4 hours and 1 minute cache_entry.time_saved = (orig_time_saved - timedelta(minutes=(60 * 4)+1)) cache_entry.save() response = cache.getCache('no_such', '/student/myuwcachetest1', {}) self.assertEquals(response, None)
def test_sws_term_policy(self): with self.settings(RESTCLIENTS_DAO_CACHE_CLASS=CACHE): cache = MyUWCache() ok_response = MockHTTP() ok_response.status = 200 ok_response.data = "xx" response = cache.getCache( 'sws', '/student/v5/term/1014,summer.json', {}) self.assertEquals(response, None) cache.processResponse( "sws", "/student/v5/term/1014,summer.json", ok_response) response = cache.getCache( 'sws', '/student/v5/term/1014,summer.json', {}) self.assertEquals(response["response"].data, 'xx') cache_entry = CacheEntryTimed.objects.get( service="sws", url="/student/v5/term/1014,summer.json") # Cached response is returned after 29 days orig_time_saved = cache_entry.time_saved cache_entry.time_saved = orig_time_saved - timedelta(days=29) cache_entry.save() response = cache.getCache( 'sws', '/student/v5/term/1014,summer.json', {}) self.assertNotEquals(response, None) # Cached response is not returned after 31 days cache_entry.time_saved = orig_time_saved - timedelta(days=31) cache_entry.save() response = cache.getCache( 'sws', '/student/v5/term/1014,summer.json', {}) self.assertEquals(response, None)
def processResponse(self, service, url, response): query = CacheEntryTimed.objects.filter(service=service, url=url) cache_entry = CacheEntryTimed() if len(query): cache_entry = query[0] if response.status == 304: if cache_entry is None: raise Exception("304, but no content??") response = MockHTTP() response.status = cache_entry.status response.data = cache_entry.content response.headers = cache_entry.headers return {"response": response} else: now = make_aware(datetime.now(), get_current_timezone()) cache_entry.service = service cache_entry.url = url cache_entry.status = response.status cache_entry.content = response.data cache_entry.headers = response.headers cache_entry.time_saved = now store_cache_entry(cache_entry) return
def _response_from_cache(self, service, url, headers, max_age_in_seconds, max_error_age=60 * 5): # If max_age_in_seconds is 0, # make sure we don't get a hit from this same second. if not max_age_in_seconds: return None now = make_aware(datetime.now(), get_current_timezone()) time_limit = now - timedelta(seconds=max_age_in_seconds) query = CacheEntryTimed.objects.filter(service=service, url=url, time_saved__gte=time_limit) if len(query): hit = query[0] if hit.status != 200 and ( now - timedelta(seconds=max_error_age) > hit.time_saved): return None response = MockHTTP() response.status = hit.status response.data = hit.content response.headers = hit.getHeaders() return { "response": response, } return None
def test_saved_headers(self): with self.settings(RESTCLIENTS_SWS_DAO_CLASS= 'restclients.dao_implementation.sws.File', RESTCLIENTS_PWS_DAO_CLASS= 'restclients.dao_implementation.pws.File', RESTCLIENTS_DAO_CACHE_CLASS= 'restclients.cache_implementation.TimeSimpleCache'): cache = TimeSimpleCache() response = MockHTTP() response.status = 200 response.data = "Cache testing" response.headers = { "link": "next,http://somewhere", "Content-type": "text", "random": "stuff", "and": "more", } cache._process_response("cache_test", "/v1/headers", response) from_db = cache._response_from_cache("cache_test", "/v1/headers", {}, 10) self.assertEquals(from_db["response"].status, 200) self.assertEquals(from_db["response"].data, "Cache testing") self.assertEquals(from_db["response"].getheader("random"), "stuff") self.assertEquals(from_db["response"].getheader("and"), "more") self.assertEquals(from_db["response"].getheader("Content-type"), "text") self.assertEquals(from_db["response"].getheader("link"), "next,http://somewhere")
def test_saved_headers(self): with self.settings(RESTCLIENTS_SWS_DAO_CLASS='restclients.dao_implementation.sws.File', RESTCLIENTS_PWS_DAO_CLASS='restclients.dao_implementation.pws.File', RESTCLIENTS_DAO_CACHE_CLASS='restclients.cache_implementation.TimeSimpleCache'): cache = TimeSimpleCache() response = MockHTTP() response.status = 200 response.data = "Cache testing" response.headers = { "link": "next,http://somewhere", "Content-type": "text", "random": "stuff", "and": "more", } cache._process_response("cache_test", "/v1/headers", response) from_db = cache._response_from_cache("cache_test", "/v1/headers", {}, 10) self.assertEquals(from_db["response"].status, 200) self.assertEquals(from_db["response"].data, "Cache testing") self.assertEquals(from_db["response"].getheader("random"), "stuff") self.assertEquals(from_db["response"].getheader("and"), "more") self.assertEquals(from_db["response"].getheader("Content-type"), "text") self.assertEquals(from_db["response"].getheader("link"), "next,http://somewhere")
def getURL(self, url, headers): if url in File._cache: response = MockHTTP() response.status = 200 response.data = File._cache[url] response.headers = {'Content-Type': 'application/json'} else: response = get_mockdata_url("irws", "file", url, headers) File._cache[url] = response.data return response
def test_myplan_default(self): with self.settings(RESTCLIENTS_DAO_CACHE_CLASS=CACHE): cache = MyUWCache() ok_response = MockHTTP() ok_response.status = 200 ok_response.data = "xx" response = cache.getCache('myplan', '/api/plan/xx', {}) self.assertEquals(response, None) cache.processResponse("myplan", "/api/plan/xx", ok_response) response = cache.getCache('myplan', '/api/plan/xx', {}) self.assertEquals(response, None)
def getURL(self, url, headers): """ This method takes a partial url, e.g. "/v1/person/123AAAAA" and a dictionary of headers. Returns an object that can be used like an HTTPResponse object. """ response = MockHTTP() response.status = 500 response.data = "" return response
def get_mockdata_url(service_name, implementation_name, url, headers): """ :param service_name: possible "sws", "pws", "book", "hfs", etc. :param implementation_name: possible values: "file", etc. """ RESOURCE_ROOT = _mockdata_path_root(service_name, implementation_name) file_path = None success = False start_time = time.time() mockdata_delay = getattr(settings, "RESTCLIENTS_MOCKDATA_DELAY", 0.0) time.sleep(mockdata_delay) for resource_dir in app_resource_dirs: logger.debug("Check url %s, service_name: %s, in resource_dir: %s" % (url, service_name, resource_dir)) response = _load_resource_from_path(resource_dir, service_name, implementation_name, url, headers) if response: request_time = time.time() - start_time rest_request.send(sender='restclients', url=url, request_time=request_time, hostname=socket.gethostname(), service_name=service_name) rest_request_passfail.send(sender='restclients', url=url, success=True, hostname=socket.gethostname(), service_name=service_name) return response # If no response has been found in any installed app, return a 404 logger.info("404 for url %s, path: %s" % (url, "resources/%s/%s/%s" % (service_name, implementation_name, convert_to_platform_safe(url)))) rest_request_passfail.send(sender='restclients', url=url, success=False, hostname=socket.gethostname(), service_name=service_name) response = MockHTTP() response.status = 404 response.reason = "Not Found" return response
def putURL(self, url, headers, body): response = MockHTTP() if body is not None: if "If-Match" in headers: response.status = 200 # update else: response.status = 201 # create response.headers = { "X-Data-Source": "GWS file mock data", "Content-Type": headers["Content-Type"] } # insert response.data time values now = int(round(time() * 1000)) for t in ['createtime', 'modifytime', 'membermodifytime']: span = '<span class="%s">' % t if not re.search(r'%s' % span, body): body = re.sub(r'(<div class="group">)', r'\1\n %s%s</span>\n' % (span, now), body) response.data = body else: response.status = 400 response.data = "Bad Request: no POST body" return response
def _process_response(self, service, url, response, overwrite_success_with_error_at=60 * 60 * 8): now = make_aware(datetime.now(), get_current_timezone()) query = CacheEntryTimed.objects.filter(service=service, url=url) cache_entry = None if len(query): cache_entry = query[0] else: cache_entry = CacheEntryTimed() if response.status != 200: # Only override a successful cache entry with an error if the # Successful entry is older than 8 hours - MUWM-509 if cache_entry.id is not None and cache_entry.status == 200: save_delta = now - cache_entry.time_saved extended_cache_delta = timedelta( seconds=overwrite_success_with_error_at) if save_delta < extended_cache_delta: response = MockHTTP() response.status = cache_entry.status response.data = cache_entry.content return {"response": response} cache_entry.service = service cache_entry.url = url cache_entry.status = response.status cache_entry.content = response.data # This extra step is needed w/ Live resources because # HTTPHeaderDict isn't serializable. header_data = {} for header in response.headers: header_data[header] = response.getheader(header) cache_entry.headers = header_data cache_entry.time_saved = now try: store_cache_entry(cache_entry) except Exception as ex: # If someone beat us in to saving a cache entry, that's ok. # We just need a very recent entry. return return
def get_mockdata_url(service_name, implementation_name, url, headers): """ :param service_name: possible "sws", "pws", "book", "hfs", etc. :param implementation_name: possible values: "file", etc. """ dir_base = dirname(__file__) __initialize_app_resource_dirs() RESOURCE_ROOT = abspath(dir_base + "/../resources/" + service_name + "/" + implementation_name) file_path = None success = False start_time = time.time() for resource_dir in app_resource_dirs: response = _load_resource_from_path(resource_dir, service_name, implementation_name, url, headers) if response: request_time = time.time() - start_time rest_request.send(sender='restclients', url=url, request_time=request_time, hostname=socket.gethostname(), service_name=service_name) rest_request_passfail.send(sender='restclients', url=url, success=True, hostname=socket.gethostname(), service_name=service_name) return response # If no response has been found in any installed app, return a 404 logger = logging.getLogger(__name__) logger.info( "404 for url %s, path: %s" % (url, "resources/%s/%s/%s" % (service_name, implementation_name, convert_to_platform_safe(url)))) rest_request_passfail.send(sender='restclients', url=url, success=False, hostname=socket.gethostname(), service_name=service_name) response = MockHTTP() response.status = 404 return response
def get_mockdata_url(service_name, implementation_name, url, headers): """ :param service_name: possible "sws", "pws", "book", "hfs", etc. :param implementation_name: possible values: "file", etc. """ dir_base = dirname(__file__) __initialize_app_resource_dirs() RESOURCE_ROOT = abspath(dir_base + "/../resources/" + service_name + "/" + implementation_name) file_path = None success = False start_time = time.time() for resource_dir in app_resource_dirs: response = _load_resource_from_path(resource_dir, service_name, implementation_name, url, headers) if response: request_time = time.time() - start_time rest_request.send(sender='restclients', url=url, request_time=request_time, hostname=socket.gethostname(), service_name=service_name) rest_request_passfail.send(sender='restclients', url=url, success=True, hostname=socket.gethostname(), service_name=service_name) return response # If no response has been found in any installed app, return a 404 logger = logging.getLogger(__name__) logger.info("404 for url %s, path: %s" % (url, "resources/%s/%s/%s" %(service_name, implementation_name, convert_to_platform_safe(url)))) rest_request_passfail.send(sender='restclients', url=url, success=False, hostname=socket.gethostname(), service_name=service_name) response = MockHTTP() response.status = 404 return response
def delete_mockdata_url(service_name, implementation_name, url, headers, dir_base=dirname(__file__)): """ :param service_name: possible "sws", "pws", "book", "hfs", etc. :param implementation_name: possible values: "file", etc. """ # Http response code 204 No Content: # The server has fulfilled the request but does not need to # return an entity-body response = MockHTTP() response.status = 204 return response
def putURL(self, url, headers, body): response = MockHTTP() if body is not None: if "If-Match" in headers: response.status = 200 # update else: response.status = 201 # create response.headers = {"X-Data-Source": "GWS file mock data", "Content-Type": headers["Content-Type"]} # insert response.data time values now = int(round(time() * 1000)) for t in ['createtime', 'modifytime', 'membermodifytime']: span = '<span class="%s">' % t if not re.search(r'%s' % span, body): body = re.sub(r'(<div class="group">)', r'\1\n %s%s</span>\n' % (span, now), body) response.data = body else: response.status = 400 response.data = "Bad Request: no POST body" return response
def post_mockdata_url(service_name, implementation_name, url, headers, body, dir_base=dirname(__file__)): """ :param service_name: possible "sws", "pws", "book", "hfs", etc. :param implementation_name: possible values: "file", etc. """ # Currently this post method does not return a response body response = MockHTTP() if body is not None: if "dispatch" in url: response.status = 200 else: response.status = 201 response.headers = { "X-Data-Source": service_name + " file mock data", "Content-Type": headers['Content-Type'] } else: response.status = 400 response.data = "Bad Request: no POST body" return response
def put_mockdata_url(service_name, implementation_name, url, headers, body, dir_base = dirname(__file__)): """ :param service_name: possible "sws", "pws", "book", "hfs", etc. :param implementation_name: possible values: "file", etc. """ #Currently this put method does not return a response body response = MockHTTP() if body is not None: response.status = 204 response.headers = {"X-Data-Source": service_name + " file mock data", "Content-Type": headers['Content-Type']} else: response.status = 400 response.data = "Bad Request: no POST body" return response
def test_myplan_default(self): with self.settings(RESTCLIENTS_DAO_CACHE_CLASS=CACHE): cache = MyUWCache() ok_response = MockHTTP() ok_response.status = 200 ok_response.data = "xx" response = cache.getCache('myplan', '/api/plan/xx', {}) self.assertEquals(response, None) cache.processResponse("myplan", "/api/plan/xx", ok_response) response = cache.getCache('myplan', '/api/plan/xx', {}) cache_entry = CacheEntryTimed.objects.get(service="myplan", url="/api/plan/xx") orig_time_saved = cache_entry.time_saved cache_entry.time_saved = (orig_time_saved - timedelta(seconds=5)) cache_entry.save() response = cache.getCache('myplan', '/api/plan/xx', {}) self.assertEquals(response, None)
def test_notice_default(self): with self.settings(RESTCLIENTS_DAO_CACHE_CLASS=CACHE): cache = MyUWCache() ok_response = MockHTTP() ok_response.status = 200 ok_response.data = "xx" response = cache.getCache('sws', '/student/v5/notice/xx', {}) self.assertEquals(response, None) cache.processResponse("sws", "/student/v5/notice/xx", ok_response) response = cache.getCache('sws', '/student/v5/notice/xx', {}) cache_entry = CacheEntryTimed.objects.get( service="sws", url="/student/v5/notice/xx") orig_time_saved = cache_entry.time_saved cache_entry.time_saved = (orig_time_saved - timedelta(minutes=60)) cache_entry.save() response = cache.getCache('sws', '/student/v5/notice/xx', {}) self.assertEquals(response, None)
def getCache(self, service, url, headers): now = make_aware(datetime.now(), get_current_timezone()) time_limit = now - timedelta(seconds=60) query = CacheEntry.objects.filter(service=service, url=url) if len(query): hit = query[0] response = MockHTTP() response.status = hit.status response.data = hit.content hit_headers = hit.getHeaders() if "ETag" in hit_headers: headers["If-None-Match"] = hit_headers["ETag"] return None
def getCache(self, service, url, headers): client = self._get_client() key = self._get_key(service, url) try: data = client.get(key) except bmemcached.exceptions.MemcachedException as ex: logger.warning("MemCached Err on get with key '%s' ==> '%s'", key, str(ex)) return if not data: return values = json.loads(data) response = MockHTTP() response.status = values["status"] response.data = values["data"] response.headers = values["headers"] return {"response": response}
def getURL(self, url, headers): if "If-None-Match" in headers and url == "/same": response = MockHTTP() response.status = 304 return response else: response = MockHTTP() response.status = 200 response.data = "Body Content" response.headers = {"ETag": "A123BBB"} return response
def test_etags(self): with self.settings(RESTCLIENTS_PWS_DAO_CLASS= 'restclients.dao_implementation.pws.ETag', RESTCLIENTS_DAO_CACHE_CLASS= 'restclients.cache_implementation.ETagCache'): # Check initial state cache = ETagCache() response = cache.getCache('pws', '/same', {}) self.assertEquals(response, None) response = cache.getCache('sws', '/same', {}) self.assertEquals(response, None) pws = PWS_DAO() initial_response = pws.getURL('/same', {}) content = initial_response.data # Make sure there's a response there after the get headers = {} hit = cache.getCache('pws', '/same', headers) self.assertEquals(hit, None) if_match = headers["If-None-Match"] self.assertEquals(if_match, "A123BBB") mock_304 = MockHTTP() mock_304.status = 304 hit = cache.processResponse('pws', '/same', mock_304) response = hit["response"] self.assertEquals(response.status, 200) self.assertEquals(response.data, content) # Make sure there's nothing for pws there after the get response = cache.getCache('sws', '/same', {}) self.assertEquals(response, None)
def test_etags(self): with self.settings(RESTCLIENTS_PWS_DAO_CLASS='restclients.dao_implementation.pws.ETag', RESTCLIENTS_DAO_CACHE_CLASS='restclients.cache_implementation.ETagCache'): # Check initial state cache = ETagCache() response = cache.getCache('pws', '/same', {}) self.assertEquals(response, None) response = cache.getCache('sws', '/same', {}) self.assertEquals(response, None) pws = PWS_DAO() initial_response = pws.getURL('/same', {}) content = initial_response.data # Make sure there's a response there after the get headers = {} hit = cache.getCache('pws', '/same', headers) self.assertEquals(hit, None) if_match = headers["If-None-Match"] self.assertEquals(if_match, "A123BBB") mock_304 = MockHTTP() mock_304.status = 304 hit = cache.processResponse('pws', '/same', mock_304) response = hit["response"] self.assertEquals(response.status, 200) self.assertEquals(response.data, content) # Make sure there's nothing for pws there after the get response = cache.getCache('sws', '/same', {}) self.assertEquals(response, None)
def test_longkeys(self): with self.settings(RESTCLIENTS_SWS_DAO_CLASS='restclients.dao_implementation.sws.File', RESTCLIENTS_DAO_CACHE_CLASS='restclients.cache_implementation.MemcachedCache'): cache = MemcachedCache() url = "".join("X" for i in range(300)) ok_response = MockHTTP() ok_response.status = 200 ok_response.data = "valid" cache.processResponse('ok', url, ok_response) # This makes sure we don't actually cache anything when the url # is too long response = cache.getCache('ok', url, {}) self.assertEquals(response, None) # But the get doesn't raise the exception w/ the set before it, # so this redundant-looking code is testing to make sure that we # catch the exception on the get as well. url = "".join("Y" for i in range(300)) response = cache.getCache('ok', url, {}) self.assertEquals(response, None)
def putURL(self, url, headers, body): response = MockHTTP() if body is not None: if "If-Match" in headers: response.status = 200 # update else: response.status = 201 # create response.headers = {"X-Data-Source": "GWS file mock data", "Content-Type": headers["Content-Type"]} response.data = body else: response.status = 400 response.data = "Bad Request: no POST body" return response
def putURL(self, url, headers, body): response = MockHTTP() logger.debug('made it to putURL') response.headers = {"Content-Type": 'application/json'} if url in File._cache: cache = json.loads(File._cache[url]) request = json.loads(body) type = request.keys()[0] for attr in request[type][0].keys(): cache[type][0][attr] = request[type][0][attr] File._cache[url] = json.dumps(cache) response.data = File._cache[url] response.status = 200 else: response.data = body response.status = 404 return response
def load(self, method, url, headers, body): response = MockHTTP() response.status = 500 response.data = "" return response
def proxy(request, service, url): user_service = UserService() actual_user = user_service.get_original_user() use_pre = False headers = {} if service == "sws": dao = SWS_DAO() headers["X-UW-Act-as"] = actual_user elif service == "pws": dao = PWS_DAO() elif service == "gws": dao = GWS_DAO() elif service == "nws": dao = NWS_DAO() elif service == "hfs": dao = Hfs_DAO() elif service == "book": dao = Book_DAO() elif service == "canvas": dao = Canvas_DAO() elif service == "grad": dao = Grad_DAO() elif service == "uwnetid": dao = Uwnetid_DAO() elif service == "libraries": dao = MyLibInfo_DAO() elif service == "libcurrics": dao = LibCurrics_DAO() elif service == "myplan": dao = MyPlan_DAO() elif service == "iasystem": dao = IASYSTEM_DAO() headers = {"Accept": "application/vnd.collection+json"} subdomain = None if url.endswith('/evaluation'): if url.startswith('uwb/') or url.startswith('uwt/'): subdomain = url[:3] url = url[4:] else: subdomain = url[:2] url = url[3:] elif service == "calendar": dao = TrumbaCalendar_DAO() use_pre = True else: return HttpResponseNotFound("Unknown service: %s" % service) url = "/%s" % quote(url) if request.GET: try: url = "%s?%s" % (url, urlencode(request.GET)) except UnicodeEncodeError: err = "Bad URL param given to the restclients browser" return HttpResponse(err) start = time() try: if service == "iasystem" and subdomain is not None: response = dao.getURL(url, headers, subdomain) else: if service == "libcurrics": if "?campus=" in url: url = url.replace("?campus=", "/") elif "course?" in url: url_prefix = re.sub(r'\?.*$', "", url) url = "%s/%s/%s/%s/%s/%s" % ( url_prefix, request.GET["year"], request.GET["quarter"], request.GET["curriculum_abbr"].replace(" ", "%20"), request.GET["course_number"], request.GET["section_id"]) response = dao.getURL(url, headers) except Exception as ex: response = MockHTTP() response.status = 500 response.data = str(ex) end = time() # Assume json, and try to format it. try: if not use_pre: content = format_json(service, response.data) json_data = response.data else: content = response.data json_data = None except Exception as e: content = format_html(service, response.data) json_data = None context = { "url": unquote(url), "content": content, "json_data": json_data, "response_code": response.status, "time_taken": "%f seconds" % (end - start), "headers": response.headers, "override_user": user_service.get_override_user(), "use_pre": use_pre, } try: loader.get_template("restclients/extra_info.html") context["has_extra_template"] = True context["extra_template"] = "restclients/extra_info.html" except TemplateDoesNotExist: pass set_wrapper_template(context) try: search_template_path = re.sub(r"\..*$", "", url) search_template = "proxy/%s%s.html" % (service, search_template_path) loader.get_template(search_template) context["search_template"] = search_template context["search"] = format_search_params(url) except TemplateDoesNotExist: context["search_template"] = None return render(request, "proxy.html", context)
def load(self, method, url, headers, body): response = MockHTTP() response.status = 404 response.data = "Not found" return response
def test_errors(self): with self.settings(RESTCLIENTS_SWS_DAO_CLASS='restclients.dao_implementation.errors.Always500', RESTCLIENTS_PWS_DAO_CLASS='restclients.dao_implementation.pws.File', RESTCLIENTS_DAO_CACHE_CLASS='restclients.cache_implementation.FourHourCache'): cache = FourHourCache() response = cache.getCache('sws', '/invalid/url', {}) self.assertEquals(response, None) sws = SWS_DAO() response = sws.getURL('/invalid/url', {}) hit = cache.getCache('sws', '/invalid/url', {}) response = hit["response"] self.assertEquals(response.status, 500) query = CacheEntryTimed.objects.filter( service="sws", url="/invalid/url", ) # Make sure that invalid entry stops being returned after 5 mintes cache_entry = query[0] cache_entry.time_saved = cache_entry.time_saved - timedelta(minutes=5) cache_entry.save() hit = cache.getCache('sws', '/invalid/url', {}) self.assertEquals(hit, None, "No hit on old, bad status codes") # Make sure bad responses don't overwrite good ones. ok_response = MockHTTP() ok_response.status = 200 ok_response.data = "xx" cache.processResponse("test", "/ok/url", ok_response) cache_response = cache.getCache("test", "/ok/url", {}) response = cache_response["response"] self.assertEquals(response.status, 200) bad_response = MockHTTP() bad_response.status = 500 bad_response.data = "This is bad data" cache.processResponse("test", "/ok/url", bad_response) cache_response = cache.getCache("test", "/ok/url", {}) response = cache_response["response"] self.assertEquals(response.status, 200) self.assertEquals(response.data, "xx") # Make sure that an old, good hit is returned when there's a fresh, # bad hit. ok_response = MockHTTP() ok_response.status = 200 ok_response.data = "valid" cache.processResponse("sws", "/valid/url", ok_response) response = sws.getURL("/valid/url", {}) self.assertEquals(response.status, 200) query = CacheEntryTimed.objects.filter( service="sws", url="/valid/url", ) cache_entry = query[0] cache_entry.time_saved = cache_entry.time_saved - timedelta(hours=5) cache_entry.save() response = sws.getURL("/valid/url", {}) self.assertEquals(response.status, 200) # But make sure eventually we stop using our cache. cache_entry.time_saved = cache_entry.time_saved - timedelta(hours=9) cache_entry.save() response = sws.getURL("/valid/url", {}) self.assertEquals(response.status, 500)
def proxy(request, service, url): if not hasattr(settings, "RESTCLIENTS_ADMIN_GROUP"): print "You must have a group defined as your admin group." print 'Configure that using RESTCLIENTS_ADMIN_GROUP="u_foo_bar"' raise Exception("Missing RESTCLIENTS_ADMIN_GROUP in settings") user_service = UserService() actual_user = user_service.get_original_user() g = Group() is_admin = g.is_member_of_group(actual_user, settings.RESTCLIENTS_ADMIN_GROUP) if not is_admin: return HttpResponseRedirect("/") use_pre = False headers = {} if service == "sws": dao = SWS_DAO() headers["X-UW-Act-as"] = actual_user elif service == "pws": dao = PWS_DAO() elif service == "gws": dao = GWS_DAO() elif service == "nws": dao = NWS_DAO() elif service == "hfs": dao = Hfs_DAO() elif service == "book": dao = Book_DAO() elif service == "canvas": dao = Canvas_DAO() elif service == "grad": dao = Grad_DAO() elif service == "uwnetid": dao = Uwnetid_DAO() elif service == "libraries": dao = MyLibInfo_DAO() elif service == "libcurrics": dao = LibCurrics_DAO() elif service == "myplan": dao = MyPlan_DAO() elif service == "iasystem": dao = IASYSTEM_DAO() headers = {"Accept": "application/vnd.collection+json"} subdomain = None if url.endswith('/evaluation'): if url.startswith('uwb/') or url.startswith('uwt/'): subdomain = url[:3] url = url[4:] else: subdomain = url[:2] url = url[3:] elif service == "calendar": dao = TrumbaCalendar_DAO() use_pre = True else: return HttpResponseNotFound("Unknown service: %s" % service) url = "/%s" % quote(url) if request.GET: try: url = "%s?%s" % (url, urlencode(request.GET)) except UnicodeEncodeError: err = "Bad URL param given to the restclients browser" return HttpResponse(err) start = time() try: if service == "iasystem" and subdomain is not None: response = dao.getURL(url, headers, subdomain) else: response = dao.getURL(url, headers) except Exception as ex: response = MockHTTP() response.status = 500 response.data = str(ex) end = time() # Assume json, and try to format it. try: if not use_pre: content = format_json(service, response.data) json_data = response.data else: content = response.data json_data = None except Exception as e: content = format_html(service, response.data) json_data = None context = { "url": unquote(url), "content": content, "json_data": json_data, "response_code": response.status, "time_taken": "%f seconds" % (end - start), "headers": response.headers, "override_user": user_service.get_override_user(), "use_pre": use_pre, } try: loader.get_template("restclients/extra_info.html") context["has_extra_template"] = True context["extra_template"] = "restclients/extra_info.html" except TemplateDoesNotExist: pass try: loader.get_template("restclients/proxy_wrapper.html") context["wrapper_template"] = "restclients/proxy_wrapper.html" except TemplateDoesNotExist: context["wrapper_template"] = "proxy_wrapper.html" try: search_template_path = re.sub(r"\..*$", "", url) search_template = "proxy/%s%s.html" % (service, search_template_path) loader.get_template(search_template) context["search_template"] = search_template context["search"] = format_search_params(url) except TemplateDoesNotExist: context["search_template"] = None return render_to_response("proxy.html", context, context_instance=RequestContext(request))
def test_errors(self): with self.settings(RESTCLIENTS_SWS_DAO_CLASS= 'restclients.dao_implementation.errors.Always500', RESTCLIENTS_PWS_DAO_CLASS= 'restclients.dao_implementation.pws.File', RESTCLIENTS_DAO_CACHE_CLASS= 'restclients.cache_implementation.FourHourCache'): cache = FourHourCache() response = cache.getCache('sws', '/invalid/url', {}) self.assertEquals(response, None) sws = SWS_DAO() response = sws.getURL('/invalid/url', {}) hit = cache.getCache('sws', '/invalid/url', {}) response = hit["response"] self.assertEquals(response.status, 500) query = CacheEntryTimed.objects.filter( service="sws", url="/invalid/url", ) # Make sure that invalid entry stops being returned after 5 mintes cache_entry = query[0] cache_entry.time_saved = cache_entry.time_saved - timedelta( minutes=5) cache_entry.save() hit = cache.getCache('sws', '/invalid/url', {}) self.assertEquals(hit, None, "No hit on old, bad status codes") # Make sure bad responses don't overwrite good ones. ok_response = MockHTTP() ok_response.status = 200 ok_response.data = "xx" cache.processResponse("test", "/ok/url", ok_response) cache_response = cache.getCache("test", "/ok/url", {}) response = cache_response["response"] self.assertEquals(response.status, 200) bad_response = MockHTTP() bad_response.status = 500 bad_response.data = "This is bad data" cache.processResponse("test", "/ok/url", bad_response) cache_response = cache.getCache("test", "/ok/url", {}) response = cache_response["response"] self.assertEquals(response.status, 200) self.assertEquals(response.data, "xx") # Make sure that an old, good hit is returned when there's a fresh, # bad hit. ok_response = MockHTTP() ok_response.status = 200 ok_response.data = "valid" cache.processResponse("sws", "/valid/url", ok_response) response = sws.getURL("/valid/url", {}) self.assertEquals(response.status, 200) query = CacheEntryTimed.objects.filter( service="sws", url="/valid/url", ) cache_entry = query[0] cache_entry.time_saved = cache_entry.time_saved - timedelta( hours=5) cache_entry.save() response = sws.getURL("/valid/url", {}) self.assertEquals(response.status, 200) # But make sure eventually we stop using our cache. cache_entry.time_saved = cache_entry.time_saved - timedelta( hours=9) cache_entry.save() response = sws.getURL("/valid/url", {}) self.assertEquals(response.status, 500)
def _load_resource_from_path(resource_dir, service_name, implementation_name, url, headers): RESOURCE_ROOT = os.path.join(resource_dir['path'], service_name, implementation_name) app = resource_dir['app'] if url == "///": # Just a placeholder to put everything else in an else. # If there are things that need dynamic work, they'd go here pass else: orig_file_path = RESOURCE_ROOT + url unquoted = unquote(orig_file_path) paths = [ convert_to_platform_safe(orig_file_path), "%s/index.html" % (convert_to_platform_safe(orig_file_path)), orig_file_path, "%s/index.html" % orig_file_path, convert_to_platform_safe(unquoted), "%s/index.html" % (convert_to_platform_safe(unquoted)), unquoted, "%s/index.html" % unquoted, ] file_path = None handle = None for path in paths: try: file_path = path handle = open(path) break except IOError as ex: pass if handle is None: return None logger = logging.getLogger(__name__) logger.debug("URL: %s; App: %s; File: %s" % (url, app, file_path)) response = MockHTTP() response.status = 200 response.data = handle.read() response.headers = {"X-Data-Source": service_name + " file mock data", } try: headers = open(handle.name + '.http-headers') file_values = json.loads(headers.read()) if "headers" in file_values: response.headers = dict(response.headers.items() + file_values['headers'].items()) if 'status' in file_values: response.status = file_values['status'] else: response.headers = dict(response.headers.items() + file_values.items()) except IOError: pass return response
def deleteURL(self, url, headers): response = MockHTTP() response.status = 200 return response
def test_sws_term_policy(self): with self.settings(RESTCLIENTS_DAO_CACHE_CLASS=CACHE): cache = MyUWCache() ok_response = MockHTTP() ok_response.status = 200 ok_response.data = "xx" response = cache.getCache('sws', '/student/v5/term/1014,summer.json', {}) self.assertEquals(response, None) cache.processResponse("sws", "/student/v5/term/1014,summer.json", ok_response) response = cache.getCache('sws', '/student/v5/term/1014,summer.json', {}) self.assertEquals(response["response"].data, 'xx') cache_entry = CacheEntryTimed.objects.get( service="sws", url="/student/v5/term/1014,summer.json") # Cached response is returned after 6 days orig_time_saved = cache_entry.time_saved cache_entry.time_saved = orig_time_saved - timedelta( minutes=(60 * 24 * 7 - 1)) cache_entry.save() response = cache.getCache('sws', '/student/v5/term/1014,summer.json', {}) self.assertNotEquals(response, None) # Cached response is not returned after 7 days cache_entry.time_saved = orig_time_saved - timedelta(days=7) cache_entry.save() response = cache.getCache('sws', '/student/v5/term/current.json', {}) self.assertEquals(response, None) response = cache.getCache('sws', '/student/v5/term/current.json', {}) self.assertEquals(response, None) cache.processResponse("sws", "/student/v5/term/current.json", ok_response) response = cache.getCache('sws', '/student/v5/term/current.json', {}) self.assertEquals(response["response"].data, 'xx') cache_entry = CacheEntryTimed.objects.get( service="sws", url="/student/v5/term/current.json") # Cached response is returned after 6 days orig_time_saved = cache_entry.time_saved cache_entry.time_saved = orig_time_saved - timedelta( minutes=(60 * 24 - 1)) cache_entry.save() response = cache.getCache('sws', '/student/v5/term/current.json', {}) self.assertNotEquals(response, None) # Cached response is not returned after 7 days cache_entry.time_saved = orig_time_saved - timedelta(days=1) cache_entry.save() response = cache.getCache('sws', '/student/v5/term/current.json', {}) self.assertEquals(response, None)
def _load_resource_from_path(resource_dir, service_name, implementation_name, url, headers): RESOURCE_ROOT = os.path.join(resource_dir['path'], service_name, implementation_name) app = resource_dir['app'] if url == "///": # Just a placeholder to put everything else in an else. # If there are things that need dynamic work, they'd go here pass else: orig_file_path = RESOURCE_ROOT + url unquoted = unquote(orig_file_path) paths = [ convert_to_platform_safe(orig_file_path), "%s/index.html" % (convert_to_platform_safe(orig_file_path)), orig_file_path, "%s/index.html" % orig_file_path, convert_to_platform_safe(unquoted), "%s/index.html" % (convert_to_platform_safe(unquoted)), unquoted, "%s/index.html" % unquoted, ] file_path = None handle = None for path in paths: try: file_path = path handle = open(path) break except IOError as ex: pass if handle is None: return None logger = logging.getLogger(__name__) logger.debug("URL: %s; App: %s; File: %s" % (url, app, file_path)) response = MockHTTP() response.status = 200 response.data = handle.read() response.headers = { "X-Data-Source": service_name + " file mock data", } try: headers = open(handle.name + '.http-headers') file_values = json.loads(headers.read()) if "headers" in file_values: response.headers = dict(response.headers.items() + file_values['headers'].items()) if 'status' in file_values: response.status = file_values['status'] else: response.headers = dict(response.headers.items() + file_values.items()) except IOError: pass return response
def putURL(self, url, headers, body): # For developing against crashes in grade submission if re.match('/student/v\d/graderoster/2013,spring,ZERROR,101,S1,', url): response = MockHTTP() response.data = "No employee found for ID 1234567890" response.status = 500 return response # Submitted too late, sad. if re.match('/student/v\d/graderoster/2013,spring,ZERROR,101,S2,', url): response = MockHTTP() response.data = "grading period not active for year/quarter" response.status = 404 return response response = MockHTTP() if body is not None: response.status = 200 response.headers = {"X-Data-Source": "SWS file mock data"} response.data = self._make_grade_roster_submitted(body) else: response.status = 400 response.data = "Bad Request: no PUT body" return response
def proxy(request, service, url): if not hasattr(settings, "RESTCLIENTS_ADMIN_GROUP"): print "You must have a group defined as your admin group." print 'Configure that using RESTCLIENTS_ADMIN_GROUP="u_foo_bar"' raise Exception("Missing RESTCLIENTS_ADMIN_GROUP in settings") user_service = UserService() actual_user = user_service.get_original_user() g = Group() is_admin = g.is_member_of_group(actual_user, settings.RESTCLIENTS_ADMIN_GROUP) if not is_admin: return HttpResponseRedirect("/") use_pre = False headers = {} if service == "sws": dao = SWS_DAO() headers["X-UW-Act-as"] = actual_user elif service == "pws": dao = PWS_DAO() elif service == "gws": dao = GWS_DAO() elif service == "nws": dao = NWS_DAO() elif service == "hfs": dao = Hfs_DAO() elif service == "book": dao = Book_DAO() elif service == "canvas": dao = Canvas_DAO() elif service == "uwnetid": dao = Uwnetid_DAO() elif service == "libraries": dao = Libraries_DAO() elif service == "myplan": dao = MyPlan_DAO() elif service == "calendar": dao = TrumbaCalendar_DAO() use_pre = True else: return HttpResponseNotFound("Unknown service: %s" % service) url = "/%s" % quote(url) if request.GET: url = "%s?%s" % (url, urlencode(request.GET)) start = time() try: response = dao.getURL(url, headers) except Exception as ex: response = MockHTTP() response.status = 500 response.data = str(ex) end = time() # Assume json, and try to format it. try: if not use_pre: content = format_json(service, response.data) json_data = response.data; else: content = response.data json_data = None except Exception as e: content = format_html(service, response.data) json_data = None; context = { "url": unquote(url), "content": content, "json_data": json_data, "response_code": response.status, "time_taken": "%f seconds" % (end - start), "headers": response.headers, "override_user": user_service.get_override_user(), "use_pre": use_pre, } try: loader.get_template("restclients/extra_info.html") context["has_extra_template"] = True context["extra_template"] = "restclients/extra_info.html" except TemplateDoesNotExist: pass try: loader.get_template("restclients/proxy_wrapper.html") context["wrapper_template"] = "restclients/proxy_wrapper.html" except TemplateDoesNotExist: context["wrapper_template"] = "proxy_wrapper.html" try: search_template_path = re.sub(r"\..*$", "", url) search_template = "proxy/%s%s.html" % (service, search_template_path) loader.get_template(search_template) context["search_template"] = search_template context["search"] = format_search_params(url) except TemplateDoesNotExist: context["search_template"] = None return render_to_response("proxy.html", context, context_instance=RequestContext(request))