def delete(cls, rhandler, guid, **kw):
helper = RequestHelper(rhandler)
try:
ArtifactAccessor.delete(guid)
helper.set_status(204)
except NotFoundException, ex:
helper.error(404)
def test_act_direct_message(self):
moxer = Mox()
api = _create_default_mocks(moxer)
_build_standard_config(moxer)
bundle = _create_actor_and_delegates(api, moxer)
actor = bundle.actor
direct_id = 1
direct = moxer.CreateMock(twitter.DirectMessage)
user = moxer.CreateMock(twitter.User)
direct.id = direct_id
direct.sender_screen_name = "mikemattozzi"
direct.text = "why is blood spattered all over your car?"
_return_direct_messages(api, [direct])
post = moxer.CreateMockAnything()
post.id = 101
_return_replies(api, ())
TwitterResponseAccessor.get_by_message_id(str(direct_id))
ArtifactAccessor.search("spattered").AndReturn(create_content_list(10))
# response
api.PostDirectMessage(direct.sender_screen_name,
IgnoreArg()).AndReturn(post)
TwitterResponseAccessor.create(str(direct.id),
response_id=str(post.id),
user=direct.sender_screen_name)
post.AsDict().AndReturn({})
moxer.ReplayAll()
actor.act()
moxer.VerifyAll()
def test_act_direct_message(self):
moxer = Mox()
api = _create_default_mocks(moxer)
_build_standard_config(moxer)
bundle = _create_actor_and_delegates(api, moxer)
actor = bundle.actor
direct_id = 1
direct = moxer.CreateMock(twitter.DirectMessage)
user = moxer.CreateMock(twitter.User)
direct.id = direct_id
direct.sender_screen_name = "mikemattozzi"
direct.text = "why is blood spattered all over your car?"
_return_direct_messages(api, [direct])
post = moxer.CreateMockAnything()
post.id = 101
_return_replies(api, ())
TwitterResponseAccessor.get_by_message_id(str(direct_id))
ArtifactAccessor.search("spattered").AndReturn(create_content_list(10))
# response
api.PostDirectMessage(direct.sender_screen_name, IgnoreArg()).AndReturn(post)
TwitterResponseAccessor.create(str(direct.id), response_id=str(post.id), user=direct.sender_screen_name)
post.AsDict().AndReturn({})
moxer.ReplayAll()
actor.act()
moxer.VerifyAll()
def test_create(self):
accessor_save_kw = self.__keywords()
source_name = accessor_save_kw['source']
content_type = accessor_save_kw['content_type']
body = accessor_save_kw['body']
self.moxer.StubOutWithMock(ArtifactInfo, "all", use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactSource, "get_or_create", use_mock_anything=True)
self.moxer.StubOutWithMock(Counters, "source_counter", use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactInfo, "create", use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactContent, "create", use_mock_anything=True)
source = MockEntity(key_name=source_name)
ArtifactInfo.all(keys_only=True).AndReturn(MockQuery(None, keys_only=True))
ArtifactSource.get_or_create(source_name).AndReturn(source)
counter = self.moxer.CreateMockAnything()
Counters.source_counter(source_name).AndReturn(counter)
counter.increment()
# TODO: I wish I could ignore keywords
md5 = ArtifactAccessor._content_md5(source_name, content_type, body)
info_save_kw = dict(source=source, source_name=source_name, content_type=content_type, content_md5=md5)
info_key = MockKey(name=self.test_id)
ArtifactInfo.create(**info_save_kw).AndReturn(info_key)
content_save_kw = dict(source=source, source_name=source_name, info=info_key, body=body)
ArtifactContent.create(info_key.name(), **content_save_kw).AndReturn(MockKey(name=self.test_id))
self.moxer.ReplayAll()
info, content, source = ArtifactAccessor.create(**accessor_save_kw)
print 'info:%s, content:%s, source:%s' % (info, content, source)
self.moxer.VerifyAll()
def test_create_no_source_name(self):
self.moxer.StubOutWithMock(ArtifactInfo, "all", use_mock_anything=True)
self.moxer.ReplayAll()
try:
ArtifactAccessor.create(msg='hi')
self.fail("exception expected.")
except IllegalArgumentException:
pass
self.moxer.VerifyAll()
def test_create_no_source_name(self):
self.moxer.StubOutWithMock(ArtifactInfo, "all", use_mock_anything=True)
self.moxer.ReplayAll()
try:
ArtifactAccessor.create(msg='hi')
self.fail("exception expected.")
except IllegalArgumentException:
pass
self.moxer.VerifyAll()
def _test_delete(self):
self.moxer.StubOutWithMock(ArtifactInfo, "get_by_guid", use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactContent, "get_by_guid", use_mock_anything=True)
guid = 'blah'
ArtifactInfo.get_by_guid(guid).AndReturn(MockEntity(MockKey(name=guid)))
ArtifactContent.get_by_guid(guid).AndReturn(MockEntity(MockKey(name=guid)))
self.moxer.ReplayAll()
ArtifactAccessor.delete(guid)
self.moxer.VerifyAll()
def test_create_duplicate(self):
self.moxer.StubOutWithMock(ArtifactInfo, "all", use_mock_anything=True)
ArtifactInfo.all(keys_only=True).AndReturn(MockQuery(range(1), keys_only=True))
self.moxer.ReplayAll()
try:
ArtifactAccessor.create(**self.__keywords())
self.fail("exception expected")
except DuplicateDataException, ex:
pass
def test_create_duplicate(self):
self.moxer.StubOutWithMock(ArtifactInfo, "all", use_mock_anything=True)
ArtifactInfo.all(keys_only=True).AndReturn(
MockQuery(range(1), keys_only=True))
self.moxer.ReplayAll()
try:
ArtifactAccessor.create(**self.__keywords())
self.fail("exception expected")
except DuplicateDataException, ex:
pass
def test_mix_response(self):
moxer = mox.Mox()
moxer.StubOutWithMock(ArtifactAccessor, "search")
inquiry = "how many elephants are in pakistan during winter"
# queries from longest word to shortest until target results are reached
ArtifactAccessor.search("elephants").AndReturn(create_content_list(5))
ArtifactAccessor.search("pakistan").AndReturn(create_content_list(5))
moxer.ReplayAll()
sources, text = new_default_mixer().mix_response(inquiry)
print text
moxer.VerifyAll()
def test_delete_nonexistent(self):
self.moxer.StubOutWithMock(ArtifactInfo, "get_by_guid", use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactContent, "get_by_guid", use_mock_anything=True)
guid = 'blah'
ArtifactInfo.get_by_guid(guid)
ArtifactContent.get_by_guid(guid)
self.moxer.ReplayAll()
try:
ArtifactAccessor.delete(guid)
self.fail("exception expected")
except NotFoundException, ex:
pass
def test_mix_response(self):
moxer = mox.Mox()
moxer.StubOutWithMock(ArtifactAccessor, "search")
inquiry = "how many elephants are in pakistan during winter"
# queries from longest word to shortest until target results are reached
ArtifactAccessor.search("elephants").AndReturn(create_content_list(5))
ArtifactAccessor.search("pakistan").AndReturn(create_content_list(5))
moxer.ReplayAll()
sources, text = new_default_mixer().mix_response(inquiry)
print text
moxer.VerifyAll()
def test_create(self):
accessor_save_kw = self.__keywords()
source_name = accessor_save_kw['source']
content_type = accessor_save_kw['content_type']
body = accessor_save_kw['body']
self.moxer.StubOutWithMock(ArtifactInfo, "all", use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactSource,
"get_or_create",
use_mock_anything=True)
self.moxer.StubOutWithMock(Counters,
"source_counter",
use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactInfo,
"create",
use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactContent,
"create",
use_mock_anything=True)
source = MockEntity(key_name=source_name)
ArtifactInfo.all(keys_only=True).AndReturn(
MockQuery(None, keys_only=True))
ArtifactSource.get_or_create(source_name).AndReturn(source)
counter = self.moxer.CreateMockAnything()
Counters.source_counter(source_name).AndReturn(counter)
counter.increment()
# TODO: I wish I could ignore keywords
md5 = ArtifactAccessor._content_md5(source_name, content_type, body)
info_save_kw = dict(source=source,
source_name=source_name,
content_type=content_type,
content_md5=md5)
info_key = MockKey(name=self.test_id)
ArtifactInfo.create(**info_save_kw).AndReturn(info_key)
content_save_kw = dict(source=source,
source_name=source_name,
info=info_key,
body=body)
ArtifactContent.create(info_key.name(), **content_save_kw).AndReturn(
MockKey(name=self.test_id))
self.moxer.ReplayAll()
info, content, source = ArtifactAccessor.create(**accessor_save_kw)
print 'info:%s, content:%s, source:%s' % (info, content, source)
self.moxer.VerifyAll()
def test_mix_response_no_search_results(self):
m = mox.Mox()
m.StubOutWithMock(ArtifactAccessor, "search")
# m.StubOutWithMock(Mixer, "random_resource")
m.StubOutWithMock(Mixer, "mix_random_limit_sources")
inquiry = "elephants"
# no search results returned
ArtifactAccessor.search("elephants").AndReturn(())
Mixer.mix_random_limit_sources(1).AndReturn(self.__default_mix_results())
# Mixer.random_resource().AndReturn(MockEntity(key_name="failblog", url="http://failblog.org"))
m.ReplayAll()
sources, text = new_default_mixer().mix_response(inquiry)
m.VerifyAll()
def test_mix_response_mix_contents_fails(self):
m = mox.Mox()
m.StubOutWithMock(ArtifactAccessor, "search")
# m.StubOutWithMock(Mixer, "random_resource")
m.StubOutWithMock(Mixer, "mix_random_limit_sources")
inquiry = "elephants"
# content should be short enough to make SentenceSpeaker fail
ArtifactAccessor.search("elephants").AndReturn(tuple(e for e in MockQuery(xrange(1), create_call=lambda id: _content(id, 0))))
Mixer.mix_random_limit_sources(1).AndReturn(self.__default_mix_results())
#Mixer.random_resource().AndReturn(MockEntity(key_name="failblog", url="http://failblog.org"))
m.ReplayAll()
sources, text = new_default_mixer().mix_response(inquiry)
m.VerifyAll()
def test_mix_response_no_search_results(self):
m = mox.Mox()
m.StubOutWithMock(ArtifactAccessor, "search")
# m.StubOutWithMock(Mixer, "random_resource")
m.StubOutWithMock(Mixer, "mix_random_limit_sources")
inquiry = "elephants"
# no search results returned
ArtifactAccessor.search("elephants").AndReturn(())
Mixer.mix_random_limit_sources(1).AndReturn(
self.__default_mix_results())
# Mixer.random_resource().AndReturn(MockEntity(key_name="failblog", url="http://failblog.org"))
m.ReplayAll()
sources, text = new_default_mixer().mix_response(inquiry)
m.VerifyAll()
def _test_delete(self):
self.moxer.StubOutWithMock(ArtifactInfo,
"get_by_guid",
use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactContent,
"get_by_guid",
use_mock_anything=True)
guid = 'blah'
ArtifactInfo.get_by_guid(guid).AndReturn(MockEntity(
MockKey(name=guid)))
ArtifactContent.get_by_guid(guid).AndReturn(
MockEntity(MockKey(name=guid)))
self.moxer.ReplayAll()
ArtifactAccessor.delete(guid)
self.moxer.VerifyAll()
def test_delete_nonexistent(self):
self.moxer.StubOutWithMock(ArtifactInfo,
"get_by_guid",
use_mock_anything=True)
self.moxer.StubOutWithMock(ArtifactContent,
"get_by_guid",
use_mock_anything=True)
guid = 'blah'
ArtifactInfo.get_by_guid(guid)
ArtifactContent.get_by_guid(guid)
self.moxer.ReplayAll()
try:
ArtifactAccessor.delete(guid)
self.fail("exception expected")
except NotFoundException, ex:
pass
def test_mix_response_mix_contents_fails(self):
m = mox.Mox()
m.StubOutWithMock(ArtifactAccessor, "search")
# m.StubOutWithMock(Mixer, "random_resource")
m.StubOutWithMock(Mixer, "mix_random_limit_sources")
inquiry = "elephants"
# content should be short enough to make SentenceSpeaker fail
ArtifactAccessor.search("elephants").AndReturn(
tuple(e
for e in MockQuery(xrange(1),
create_call=lambda id: _content(id, 0))))
Mixer.mix_random_limit_sources(1).AndReturn(
self.__default_mix_results())
#Mixer.random_resource().AndReturn(MockEntity(key_name="failblog", url="http://failblog.org"))
m.ReplayAll()
sources, text = new_default_mixer().mix_response(inquiry)
m.VerifyAll()
def get(self, **kw):
helper = RequestHelper(self)
start = int(self.request.get("start", 0))
count = int(self.request.get("count", 10))
q = ArtifactInfo.all().order("-modified")
json_results = []
if q.count():
for a_info in q.fetch(count, start):
a_content = ArtifactAccessor.get_content_by_guid(a_info.guid)
json_results.append(ArtifactsHelper.artifact_to_hash(a_info, a_content))
helper.write_json(json_results)
def search(request):
t = loader.get_template("search.html")
q = request.REQUEST.get("q", None)
contents = None
if not q:
msg = "no query provided"
else:
q = urllib.unquote(q)
contents = ArtifactAccessor.search(q)
msg = "%d results found for query '%s'" % (len(contents), q)
c = Context(dict(msg=msg, contents=contents))
return HttpResponse(t.render(c))
class ArtifactsHelper:
@classmethod
def post(cls, request_handler, **kw):
helper = RequestHelper(request_handler)
request = request_handler.request
username = kw.get("username", None)
user = User(username) if username else users.get_current_user()
json_body = request.body
if not json_body:
helper.error(400, "body required")
return
decoded_body = urllib.unquote(json_body)
try:
artifact_hash = json.loads(decoded_body)
except json.JSONDecodeError, e:
msg = "malformed json: %s" % decoded_body
helper.error(400, msg)
logging.info(msg)
return
# de-unicodes keys
decoded_hash = {}
for k, v in artifact_hash.iteritems():
decoded_hash[k.encode("utf-8")] = v
fields = ("source", "content-type", "body")
result = Hashes.fetch_fields(decoded_hash, fields)
if result.missing_fields:
msg = "missing fields: %s" % result.missing_fields
helper.error(400, msg)
logging.info(msg)
return
source, content_type, content_body = result.values
# name of info_key is guid
try:
info_key, src_key, content_key = ArtifactAccessor.create(source=source,
content_type=content_type,
body=content_body,
modified_by=user)
guid = info_key.name()
helper.set_status(204)
location = cls.artifact_uri(request, guid)
helper.header("Location", location)
except DuplicateDataException, ex:
helper.error(409, ex.message)
def mix_response(self, message, min_results=10, **kw):
# find longest word in message
words = [w for w in message.split()]
sorted_words = sorted(words, key=lambda w: len(w), reverse=True)
# searches from longest word to shortest until result count > min result threshold
contents = []
for w in sorted_words:
results = ArtifactAccessor.search(w)
logging.debug("search for '%s' found %d result(s)" % (w, len(results)))
contents.extend(results)
if len(contents) >= min_results:
break
are_results_mixed = False
if contents:
# mix search results
try:
result = self.__mix_contents(contents, **kw)
are_results_mixed = True
except MissingDataException, e:
logging.error(traceback.print_exc())
def mix_response(self, message, min_results=10, **kw):
# find longest word in message
words = [w for w in message.split()]
sorted_words = sorted(words, key=lambda w: len(w), reverse=True)
# searches from longest word to shortest until result count > min result threshold
contents = []
for w in sorted_words:
results = ArtifactAccessor.search(w)
logging.debug("search for '%s' found %d result(s)" %
(w, len(results)))
contents.extend(results)
if len(contents) >= min_results:
break
are_results_mixed = False
if contents:
# mix search results
try:
result = self.__mix_contents(contents, **kw)
are_results_mixed = True
except MissingDataException, e:
logging.error(traceback.print_exc())
def ingest_feed_entries(feed, user, error_call=None):
"""
yields:
(artifact guid, entry) tuple
"""
# TODO: use etag from previous ingest
for entry in feeds.generate_feed_entries(feed.url):
try:
stripped_content = entry.get("stripped_content")
if stripped_content:
# ensures this is a non-empty entry
link = entry.get("link")
raw_modified = entry.get("modified")
if raw_modified:
modified = datetime(*raw_modified[0:-2])
else:
modified = None
logging.debug("%s modified %s (%s)" %
(link, modified, modified.__class__))
url_resource = UrlResourceAccessor.get_or_create(
link, source_modified=modified, feed=feed)
# TODO: check if there is already an artifact for this resource
info_key, content_key, source_key, created = ArtifactAccessor.find_or_create(
source=feed.artifact_source.name,
content_type="text/plain",
body=stripped_content,
url=link,
url_resource=url_resource,
modified_by=user)
yield info_key.name(), entry, created
except Exception, e:
if error_call:
error_call(entry, e)
else:
raise e
def ingest_feed_entries(feed, user, error_call=None):
"""
yields:
(artifact guid, entry) tuple
"""
# TODO: use etag from previous ingest
for entry in feeds.generate_feed_entries(feed.url):
try:
stripped_content = entry.get("stripped_content")
if stripped_content:
# ensures this is a non-empty entry
link = entry.get("link")
raw_modified = entry.get("modified")
if raw_modified:
modified = datetime(*raw_modified[0:-2])
else:
modified = None
logging.debug("%s modified %s (%s)" % (link, modified, modified.__class__))
url_resource = UrlResourceAccessor.get_or_create(link,
source_modified=modified,
feed=feed)
# TODO: check if there is already an artifact for this resource
info_key, content_key, source_key, created = ArtifactAccessor.find_or_create(source=feed.artifact_source.name,
content_type="text/plain",
body=stripped_content,
url = link,
url_resource=url_resource,
modified_by=user)
yield info_key.name(), entry, created
except Exception, e:
if error_call:
error_call(entry, e)
else:
raise e
def test_create_empty(self):
try:
ArtifactAccessor.create()
self.fail("exception expected")
except IllegalArgumentException, ex:
pass
def _search_results(term, results):
ArtifactAccessor.search(term).AndReturn(results)
def mix_after_search(self, term, **kw):
contents = ArtifactAccessor.search(term)
return self.__mix_contents(contents, **kw)
def mix_after_search(self, term, **kw):
contents = ArtifactAccessor.search(term)
return self.__mix_contents(contents, **kw)
def _search_results(term, results):
ArtifactAccessor.search(term).AndReturn(results)
def test_create_empty(self):
try:
ArtifactAccessor.create()
self.fail("exception expected")
except IllegalArgumentException, ex:
pass