def _test_post_no_user(self): moxer = Mox() request, response = new_mock_request_response(moxer) moxer.StubOutWithMock(users, "get_current_user", use_mock_anything=True) moxer.StubOutWithMock(User, "__init__", use_mock_anything=True) moxer.StubOutWithMock(Feed, "get_by_source_name", use_mock_anything=True) moxer.StubOutWithMock(model, "ingest_feed_entries") source_name = "hi" username = Services.API_USER user = MockEntity(key_name=username, email=lambda: username) # passes auth (via cron) users.get_current_user().AndReturn(user) handler = IngestHandler() # no logged in user users.get_current_user() User.__init__(username) handler.initialize(request, response) feed = MockEntity(key_name=source_name, url="no") Feed.get_by_source_name(source_name, return_none=True).AndReturn(feed) model.ingest_feed_entries(feed, None, error_call=IgnoreArg()).AndReturn(()) moxer.ReplayAll() handler.post(source_name) moxer.VerifyAll()
def put(self, source_name): helper = RequestHelper(self) source_name = urllib.unquote(source_name) success, values = read_json_fields(helper, "url", "active", logger=logging) if not success: return url, active = values # a Feed must be sole owner of an ArtifactSource; # fails if source already exists and is already linked to a feed source = ArtifactSourceAccessor.get_by_name(source_name, return_none=True) if source: source_feed_key = Feed.get_by_source(source, keys_only=True, return_none=True) if source_feed_key: msg = "source '%s' is referenced by feed %s" % (source_name, source_feed_key.name()) helper.error(409, msg) else: source = ArtifactSourceAccessor.create(source_name) # creates UrlResource if necessary resource = UrlResourceAccessor.get_by_url(url, return_none=True) if not resource: resource = UrlResourceAccessor.create(url) # create or update Feed feed = Feed.get_by_source_name(source_name, return_none=True) if feed: feed.artifact_source = source feed.url_resource = resource feed.put() else: Feed.create(source_name, artifact_source=source, url=url, url_resource=resource, active=bool(active)) helper.set_status(204)
def test_post_handles_ingest_error(self): m = Mox() request, response = new_mock_request_response(m) m.StubOutWithMock(Feed, "get_by_source_name" ) request.get("keep").AndReturn(None) feed_name = "blah" Feed.get_by_source_name(feed_name, return_none=True).AndRaise(Exception("real bad")) # response.set_status(200) m.ReplayAll() handler = CronIngestHandler() handler.initialize(request, response) _assert_handles_error(lambda: handler.post(feed_name)) m.VerifyAll()
def get(self, source_name): helper = RequestHelper(self) source_name = urllib.unquote(source_name) f = Feed.get_by_source_name(source_name, return_none=True) if not f: helper.error(404) return helper.write_json(build_feed_hash(f))
def test_post_handles_ingest_error(self): m = Mox() request, response = new_mock_request_response(m) m.StubOutWithMock(Feed, "get_by_source_name") request.get("keep").AndReturn(None) feed_name = "blah" Feed.get_by_source_name(feed_name, return_none=True).AndRaise( Exception("real bad")) # response.set_status(200) m.ReplayAll() handler = CronIngestHandler() handler.initialize(request, response) _assert_handles_error(lambda: handler.post(feed_name)) m.VerifyAll()
def delete(self, source_name): helper = RequestHelper(self) source_name = urllib.unquote(source_name) feed = Feed.get_by_source_name(source_name, return_none=True) if not feed: helper.error(404) return feed.delete() helper.set_status(204)
def ingest(cls, handler, source_name): helper = RequestHelper(handler) source_name = urllib.unquote(source_name) keep = handler.request.get("keep") if keep: keep = int(keep) else: keep = 50 # TODO: get from cache f = Feed.get_by_source_name(source_name, return_none=True) if not f: helper.error(404) return results = {} entries = [] results['created'] = entries # TODO: use etag from previous ingest error_call = lambda entry, ex: logging.error(Exceptions.format_last()) user = users.get_current_user() if not user: # there is no logged in user for cron requests user = User(Services.API_USER) try: for artifact_guid, entry, created in model.ingest_feed_entries( f, user, error_call=error_call): entries.append({ "artifact-guid": artifact_guid, "url": entry.link, "title": entry.title, "created": created }) finally: # delete oldest feed entries # TODO: shouldn't I be deleting ArtifactContent instances also? def delete_info(c): try: i = c.info if i: i.delete() except Exception, e: pass deleted_key_names = ArtifactContent.delete_oldest_by_source( f.artifact_source, keep, pre_call=delete_info) results['deleted'] = deleted_key_names Counters.source_counter(f.artifact_source.name).decrement( len(deleted_key_names))
def ingest(cls, handler, source_name): helper = RequestHelper(handler) source_name = urllib.unquote(source_name) keep = handler.request.get("keep") if keep: keep = int(keep) else: keep = 50 # TODO: get from cache f = Feed.get_by_source_name(source_name, return_none=True) if not f: helper.error(404) return results = {} entries = [] results['created'] = entries # TODO: use etag from previous ingest error_call = lambda entry, ex: logging.error(Exceptions.format_last()) user = users.get_current_user() if not user: # there is no logged in user for cron requests user = User(Services.API_USER) try: for artifact_guid, entry, created in model.ingest_feed_entries(f, user, error_call=error_call): entries.append({ "artifact-guid": artifact_guid, "url": entry.link, "title": entry.title, "created": created }) finally: # delete oldest feed entries # TODO: shouldn't I be deleting ArtifactContent instances also? def delete_info(c): try: i = c.info if i: i.delete() except Exception, e: pass deleted_key_names = ArtifactContent.delete_oldest_by_source(f.artifact_source, keep, pre_call=delete_info) results['deleted'] = deleted_key_names Counters.source_counter(f.artifact_source.name).decrement(len(deleted_key_names))
def test_post_with_user(self): moxer = Mox() request, response = new_mock_request_response(moxer) moxer.StubOutWithMock(users, "get_current_user", use_mock_anything=True) # moxer.StubOutWithMock(ArtifactInfo, "delete_oldest_by_source", use_mock_anything=True) moxer.StubOutWithMock(ArtifactContent, "delete_oldest_by_source") moxer.StubOutWithMock(Counters, "source_counter") moxer.StubOutWithMock(Feed, "get_by_source_name", use_mock_anything=True) moxer.StubOutWithMock(model, "ingest_feed_entries") source_name = "hi" username = Services.API_USER user = MockEntity(key_name=username, email=lambda: username) users.get_current_user().AndReturn(user) handler = IngestHandler() users.get_current_user().AndReturn(user) handler.initialize(request, response) request.get("keep").AndReturn(None) counter = moxer.CreateMock(Counter) Counters.source_counter(source_name).AndReturn(counter) counter.decrement(IgnoreArg()) source = MockEntity(key_name=source_name, name=source_name) feed = MockEntity(key_name=source_name, url="no", artifact_source=source) # ArtifactInfo.delete_oldest_by_source(source, IgnoreArg()).AndReturn([]) ArtifactContent.delete_oldest_by_source(source, IgnoreArg(), pre_call=IgnoreArg()).AndReturn([]) Feed.get_by_source_name(source_name, return_none=True).AndReturn(feed) model.ingest_feed_entries(feed, user, error_call=IgnoreArg()).AndReturn(()) moxer.ReplayAll() handler.post(source_name) moxer.VerifyAll()
def get_by_source_name(cls, source_name, **kw): return Feed.get_by_source_name(source_name, **kw)