def _test_post_no_user(self):
moxer = Mox()
request, response = new_mock_request_response(moxer)
moxer.StubOutWithMock(users, "get_current_user", use_mock_anything=True)
moxer.StubOutWithMock(User, "__init__", use_mock_anything=True)
moxer.StubOutWithMock(Feed, "get_by_source_name", use_mock_anything=True)
moxer.StubOutWithMock(model, "ingest_feed_entries")
source_name = "hi"
username = Services.API_USER
user = MockEntity(key_name=username, email=lambda: username)
# passes auth (via cron)
users.get_current_user().AndReturn(user)
handler = IngestHandler()
# no logged in user
users.get_current_user()
User.__init__(username)
handler.initialize(request, response)
feed = MockEntity(key_name=source_name, url="no")
Feed.get_by_source_name(source_name, return_none=True).AndReturn(feed)
model.ingest_feed_entries(feed, None, error_call=IgnoreArg()).AndReturn(())
moxer.ReplayAll()
handler.post(source_name)
moxer.VerifyAll()
def put(self, source_name):
helper = RequestHelper(self)
source_name = urllib.unquote(source_name)
success, values = read_json_fields(helper, "url", "active", logger=logging)
if not success:
return
url, active = values
# a Feed must be sole owner of an ArtifactSource;
# fails if source already exists and is already linked to a feed
source = ArtifactSourceAccessor.get_by_name(source_name, return_none=True)
if source:
source_feed_key = Feed.get_by_source(source, keys_only=True, return_none=True)
if source_feed_key:
msg = "source '%s' is referenced by feed %s" % (source_name, source_feed_key.name())
helper.error(409, msg)
else:
source = ArtifactSourceAccessor.create(source_name)
# creates UrlResource if necessary
resource = UrlResourceAccessor.get_by_url(url, return_none=True)
if not resource:
resource = UrlResourceAccessor.create(url)
# create or update Feed
feed = Feed.get_by_source_name(source_name, return_none=True)
if feed:
feed.artifact_source = source
feed.url_resource = resource
feed.put()
else:
Feed.create(source_name, artifact_source=source, url=url, url_resource=resource, active=bool(active))
helper.set_status(204)
def test_post_handles_ingest_error(self):
m = Mox()
request, response = new_mock_request_response(m)
m.StubOutWithMock(Feed, "get_by_source_name" )
request.get("keep").AndReturn(None)
feed_name = "blah"
Feed.get_by_source_name(feed_name, return_none=True).AndRaise(Exception("real bad"))
# response.set_status(200)
m.ReplayAll()
handler = CronIngestHandler()
handler.initialize(request, response)
_assert_handles_error(lambda: handler.post(feed_name))
m.VerifyAll()
def get(self, source_name):
helper = RequestHelper(self)
source_name = urllib.unquote(source_name)
f = Feed.get_by_source_name(source_name, return_none=True)
if not f:
helper.error(404)
return
helper.write_json(build_feed_hash(f))
def test_post_handles_ingest_error(self):
m = Mox()
request, response = new_mock_request_response(m)
m.StubOutWithMock(Feed, "get_by_source_name")
request.get("keep").AndReturn(None)
feed_name = "blah"
Feed.get_by_source_name(feed_name, return_none=True).AndRaise(
Exception("real bad"))
# response.set_status(200)
m.ReplayAll()
handler = CronIngestHandler()
handler.initialize(request, response)
_assert_handles_error(lambda: handler.post(feed_name))
m.VerifyAll()
def delete(self, source_name):
helper = RequestHelper(self)
source_name = urllib.unquote(source_name)
feed = Feed.get_by_source_name(source_name, return_none=True)
if not feed:
helper.error(404)
return
feed.delete()
helper.set_status(204)
def ingest(cls, handler, source_name):
helper = RequestHelper(handler)
source_name = urllib.unquote(source_name)
keep = handler.request.get("keep")
if keep:
keep = int(keep)
else:
keep = 50
# TODO: get from cache
f = Feed.get_by_source_name(source_name, return_none=True)
if not f:
helper.error(404)
return
results = {}
entries = []
results['created'] = entries
# TODO: use etag from previous ingest
error_call = lambda entry, ex: logging.error(Exceptions.format_last())
user = users.get_current_user()
if not user:
# there is no logged in user for cron requests
user = User(Services.API_USER)
try:
for artifact_guid, entry, created in model.ingest_feed_entries(
f, user, error_call=error_call):
entries.append({
"artifact-guid": artifact_guid,
"url": entry.link,
"title": entry.title,
"created": created
})
finally:
# delete oldest feed entries
# TODO: shouldn't I be deleting ArtifactContent instances also?
def delete_info(c):
try:
i = c.info
if i:
i.delete()
except Exception, e:
pass
deleted_key_names = ArtifactContent.delete_oldest_by_source(
f.artifact_source, keep, pre_call=delete_info)
results['deleted'] = deleted_key_names
Counters.source_counter(f.artifact_source.name).decrement(
len(deleted_key_names))
def ingest(cls, handler, source_name):
helper = RequestHelper(handler)
source_name = urllib.unquote(source_name)
keep = handler.request.get("keep")
if keep:
keep = int(keep)
else:
keep = 50
# TODO: get from cache
f = Feed.get_by_source_name(source_name, return_none=True)
if not f:
helper.error(404)
return
results = {}
entries = []
results['created'] = entries
# TODO: use etag from previous ingest
error_call = lambda entry, ex: logging.error(Exceptions.format_last())
user = users.get_current_user()
if not user:
# there is no logged in user for cron requests
user = User(Services.API_USER)
try:
for artifact_guid, entry, created in model.ingest_feed_entries(f, user, error_call=error_call):
entries.append({ "artifact-guid": artifact_guid,
"url": entry.link,
"title": entry.title,
"created": created })
finally:
# delete oldest feed entries
# TODO: shouldn't I be deleting ArtifactContent instances also?
def delete_info(c):
try:
i = c.info
if i:
i.delete()
except Exception, e:
pass
deleted_key_names = ArtifactContent.delete_oldest_by_source(f.artifact_source, keep, pre_call=delete_info)
results['deleted'] = deleted_key_names
Counters.source_counter(f.artifact_source.name).decrement(len(deleted_key_names))
def test_post_with_user(self):
moxer = Mox()
request, response = new_mock_request_response(moxer)
moxer.StubOutWithMock(users, "get_current_user", use_mock_anything=True)
# moxer.StubOutWithMock(ArtifactInfo, "delete_oldest_by_source", use_mock_anything=True)
moxer.StubOutWithMock(ArtifactContent, "delete_oldest_by_source")
moxer.StubOutWithMock(Counters, "source_counter")
moxer.StubOutWithMock(Feed, "get_by_source_name", use_mock_anything=True)
moxer.StubOutWithMock(model, "ingest_feed_entries")
source_name = "hi"
username = Services.API_USER
user = MockEntity(key_name=username, email=lambda: username)
users.get_current_user().AndReturn(user)
handler = IngestHandler()
users.get_current_user().AndReturn(user)
handler.initialize(request, response)
request.get("keep").AndReturn(None)
counter = moxer.CreateMock(Counter)
Counters.source_counter(source_name).AndReturn(counter)
counter.decrement(IgnoreArg())
source = MockEntity(key_name=source_name, name=source_name)
feed = MockEntity(key_name=source_name, url="no", artifact_source=source)
# ArtifactInfo.delete_oldest_by_source(source, IgnoreArg()).AndReturn([])
ArtifactContent.delete_oldest_by_source(source, IgnoreArg(), pre_call=IgnoreArg()).AndReturn([])
Feed.get_by_source_name(source_name, return_none=True).AndReturn(feed)
model.ingest_feed_entries(feed, user, error_call=IgnoreArg()).AndReturn(())
moxer.ReplayAll()
handler.post(source_name)
moxer.VerifyAll()
def get_by_source_name(cls, source_name, **kw):
return Feed.get_by_source_name(source_name, **kw)
def get_by_source_name(cls, source_name, **kw):
return Feed.get_by_source_name(source_name, **kw)