def test_create(self):
accessor_save_kw = self.__keywords()
source_name = accessor_save_kw['source']
content_type = accessor_save_kw['content_type']
body = accessor_save_kw['body']
self.moxer.StubOutWithMock(ArtifactInfo, "all", use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactSource, "get_or_create", use_mock_anything=True)
self.moxer.StubOutWithMock(Counters, "source_counter", use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactInfo, "create", use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactContent, "create", use_mock_anything=True)
source = MockEntity(key_name=source_name)
ArtifactInfo.all(keys_only=True).AndReturn(MockQuery(None, keys_only=True))
ArtifactSource.get_or_create(source_name).AndReturn(source)
counter = self.moxer.CreateMockAnything()
Counters.source_counter(source_name).AndReturn(counter)
counter.increment()
# TODO: I wish I could ignore keywords
md5 = ArtifactAccessor._content_md5(source_name, content_type, body)
info_save_kw = dict(source=source, source_name=source_name, content_type=content_type, content_md5=md5)
info_key = MockKey(name=self.test_id)
ArtifactInfo.create(**info_save_kw).AndReturn(info_key)
content_save_kw = dict(source=source, source_name=source_name, info=info_key, body=body)
ArtifactContent.create(info_key.name(), **content_save_kw).AndReturn(MockKey(name=self.test_id))
self.moxer.ReplayAll()
info, content, source = ArtifactAccessor.create(**accessor_save_kw)
print 'info:%s, content:%s, source:%s' % (info, content, source)
self.moxer.VerifyAll()
def ingest(cls, handler, source_name):
helper = RequestHelper(handler)
source_name = urllib.unquote(source_name)
keep = handler.request.get("keep")
if keep:
keep = int(keep)
else:
keep = 50
# TODO: get from cache
f = Feed.get_by_source_name(source_name, return_none=True)
if not f:
helper.error(404)
return
results = {}
entries = []
results['created'] = entries
# TODO: use etag from previous ingest
error_call = lambda entry, ex: logging.error(Exceptions.format_last())
user = users.get_current_user()
if not user:
# there is no logged in user for cron requests
user = User(Services.API_USER)
try:
for artifact_guid, entry, created in model.ingest_feed_entries(
f, user, error_call=error_call):
entries.append({
"artifact-guid": artifact_guid,
"url": entry.link,
"title": entry.title,
"created": created
})
finally:
# delete oldest feed entries
# TODO: shouldn't I be deleting ArtifactContent instances also?
def delete_info(c):
try:
i = c.info
if i:
i.delete()
except Exception, e:
pass
deleted_key_names = ArtifactContent.delete_oldest_by_source(
f.artifact_source, keep, pre_call=delete_info)
results['deleted'] = deleted_key_names
Counters.source_counter(f.artifact_source.name).decrement(
len(deleted_key_names))
def test_create(self):
accessor_save_kw = self.__keywords()
source_name = accessor_save_kw['source']
content_type = accessor_save_kw['content_type']
body = accessor_save_kw['body']
self.moxer.StubOutWithMock(ArtifactInfo, "all", use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactSource,
"get_or_create",
use_mock_anything=True)
self.moxer.StubOutWithMock(Counters,
"source_counter",
use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactInfo,
"create",
use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactContent,
"create",
use_mock_anything=True)
source = MockEntity(key_name=source_name)
ArtifactInfo.all(keys_only=True).AndReturn(
MockQuery(None, keys_only=True))
ArtifactSource.get_or_create(source_name).AndReturn(source)
counter = self.moxer.CreateMockAnything()
Counters.source_counter(source_name).AndReturn(counter)
counter.increment()
# TODO: I wish I could ignore keywords
md5 = ArtifactAccessor._content_md5(source_name, content_type, body)
info_save_kw = dict(source=source,
source_name=source_name,
content_type=content_type,
content_md5=md5)
info_key = MockKey(name=self.test_id)
ArtifactInfo.create(**info_save_kw).AndReturn(info_key)
content_save_kw = dict(source=source,
source_name=source_name,
info=info_key,
body=body)
ArtifactContent.create(info_key.name(), **content_save_kw).AndReturn(
MockKey(name=self.test_id))
self.moxer.ReplayAll()
info, content, source = ArtifactAccessor.create(**accessor_save_kw)
print 'info:%s, content:%s, source:%s' % (info, content, source)
self.moxer.VerifyAll()
def ingest(cls, handler, source_name):
helper = RequestHelper(handler)
source_name = urllib.unquote(source_name)
keep = handler.request.get("keep")
if keep:
keep = int(keep)
else:
keep = 50
# TODO: get from cache
f = Feed.get_by_source_name(source_name, return_none=True)
if not f:
helper.error(404)
return
results = {}
entries = []
results['created'] = entries
# TODO: use etag from previous ingest
error_call = lambda entry, ex: logging.error(Exceptions.format_last())
user = users.get_current_user()
if not user:
# there is no logged in user for cron requests
user = User(Services.API_USER)
try:
for artifact_guid, entry, created in model.ingest_feed_entries(f, user, error_call=error_call):
entries.append({ "artifact-guid": artifact_guid,
"url": entry.link,
"title": entry.title,
"created": created })
finally:
# delete oldest feed entries
# TODO: shouldn't I be deleting ArtifactContent instances also?
def delete_info(c):
try:
i = c.info
if i:
i.delete()
except Exception, e:
pass
deleted_key_names = ArtifactContent.delete_oldest_by_source(f.artifact_source, keep, pre_call=delete_info)
results['deleted'] = deleted_key_names
Counters.source_counter(f.artifact_source.name).decrement(len(deleted_key_names))
def post(self, **kw):
helper = RequestHelper(self)
results = {}
source_q = ArtifactSource.all()
for s in source_q:
artifact_q = ArtifactInfo.find_by_source(s)
count = len([a for a in artifact_q])
counter = Counters.source_counter(s.name)
old_count = counter.count()
counter.set(count)
source_result = {'old': old_count}
# if source is linked to a feed, I can't delete it
feed = Feed.get_by_source(s, return_none=True)
if feed:
source_result['feed'] = feed.url
if not count and not feed:
s.delete()
source_result['deleted'] = True
if count:
source_result['new'] = count
results[s.name] = source_result
helper.write_json(results)
def post(self, **kw):
helper = RequestHelper(self)
results = {}
source_q = ArtifactSource.all()
for s in source_q:
artifact_q = ArtifactInfo.find_by_source(s)
count = len([a for a in artifact_q])
counter = Counters.source_counter(s.name)
old_count = counter.count()
counter.set(count)
source_result = { 'old': old_count }
# if source is linked to a feed, I can't delete it
feed = Feed.get_by_source(s, return_none=True)
if feed:
source_result['feed'] = feed.url
if not count and not feed:
s.delete()
source_result['deleted'] = True
if count:
source_result['new'] = count
results[s.name] = source_result
helper.write_json(results)
def source_hash(source):
c = Counters.source_counter(source.name)
return { 'name': source.name,
'count': c.count(),
'content-count': ArtifactSourceAccessor.count_content(source),
'info-count': ArtifactSourceAccessor.count_infos(source)
}
def source_hash(source):
c = Counters.source_counter(source.name)
return {
'name': source.name,
'count': c.count(),
'content-count': ArtifactSourceAccessor.count_content(source),
'info-count': ArtifactSourceAccessor.count_infos(source)
}
def test_post_with_user(self):
moxer = Mox()
request, response = new_mock_request_response(moxer)
moxer.StubOutWithMock(users, "get_current_user", use_mock_anything=True)
# moxer.StubOutWithMock(ArtifactInfo, "delete_oldest_by_source", use_mock_anything=True)
moxer.StubOutWithMock(ArtifactContent, "delete_oldest_by_source")
moxer.StubOutWithMock(Counters, "source_counter")
moxer.StubOutWithMock(Feed, "get_by_source_name", use_mock_anything=True)
moxer.StubOutWithMock(model, "ingest_feed_entries")
source_name = "hi"
username = Services.API_USER
user = MockEntity(key_name=username, email=lambda: username)
users.get_current_user().AndReturn(user)
handler = IngestHandler()
users.get_current_user().AndReturn(user)
handler.initialize(request, response)
request.get("keep").AndReturn(None)
counter = moxer.CreateMock(Counter)
Counters.source_counter(source_name).AndReturn(counter)
counter.decrement(IgnoreArg())
source = MockEntity(key_name=source_name, name=source_name)
feed = MockEntity(key_name=source_name, url="no", artifact_source=source)
# ArtifactInfo.delete_oldest_by_source(source, IgnoreArg()).AndReturn([])
ArtifactContent.delete_oldest_by_source(source, IgnoreArg(), pre_call=IgnoreArg()).AndReturn([])
Feed.get_by_source_name(source_name, return_none=True).AndReturn(feed)
model.ingest_feed_entries(feed, user, error_call=IgnoreArg()).AndReturn(())
moxer.ReplayAll()
handler.post(source_name)
moxer.VerifyAll()