def test_mix_sources_1(self):
moxer = mox.Mox()
moxer.StubOutWithMock(ArtifactSource, "get_multiple_by_name")
moxer.StubOutWithMock(ArtifactContent, "all")
source_name = "source/1"
source = MockEntity(key_name=source_name, name=source_name)
def _content(id):
id_str = str(id)
return MockEntity(key_name=id_str,
guid=id_str,
body=generate_phrase(5),
source=source,
source_name=source.name)
ArtifactSource.get_multiple_by_name(source_name).AndReturn((source, ))
ArtifactContent.all().AndReturn(
MockQuery(xrange(12), create_call=_content))
moxer.ReplayAll()
sources, text = new_default_mixer().mix_sources(source_name)
self.assertEquals(len(sources), 1)
moxer.VerifyAll()
def test_mix_sources_2(self):
moxer = mox.Mox()
moxer.StubOutWithMock(ArtifactSource, "get_multiple_by_name")
moxer.StubOutWithMock(ArtifactContent, "all")
source_names = ("source/1", "source/2")
sources = [
MockEntity(key_name=name, name=name) for name in source_names
]
def _content(id):
# toggles between sources
source = sources[id % 2]
id_str = str(id)
return MockEntity(key_name=id_str,
guid=id_str,
body=generate_phrase(5),
source=source,
source_name=source.name)
ArtifactSource.get_multiple_by_name(source_names).AndReturn(sources)
# performs ArtifactContent query for each source
ArtifactContent.all().AndReturn(
MockQuery(xrange(12), create_call=_content))
ArtifactContent.all().AndReturn(
MockQuery(xrange(12), create_call=_content))
moxer.ReplayAll()
mixed_sources, text = new_default_mixer().mix_sources(source_names)
self.assertEquals(len(mixed_sources), len(source_names))
moxer.VerifyAll()
def test_mix_sources_2(self):
moxer = mox.Mox()
moxer.StubOutWithMock(ArtifactSource, "get_multiple_by_name")
moxer.StubOutWithMock(ArtifactContent, "all")
source_names = ("source/1", "source/2")
sources = [MockEntity(key_name=name, name=name) for name in source_names]
def _content(id):
# toggles between sources
source = sources[id % 2]
id_str = str(id)
return MockEntity(key_name=id_str, guid=id_str, body=generate_phrase(5), source=source, source_name=source.name)
ArtifactSource.get_multiple_by_name(source_names).AndReturn(sources)
# performs ArtifactContent query for each source
ArtifactContent.all().AndReturn(MockQuery(xrange(12), create_call=_content))
ArtifactContent.all().AndReturn(MockQuery(xrange(12), create_call=_content))
moxer.ReplayAll()
mixed_sources, text = new_default_mixer().mix_sources(source_names)
self.assertEquals(len(mixed_sources), len(source_names))
moxer.VerifyAll()
def test_mix_random_limit_sources_1(self):
moxer = mox.Mox()
moxer.StubOutWithMock(ArtifactSource, "all")
moxer.StubOutWithMock(ArtifactContent, "all")
ArtifactSource.all().AndReturn(MockQuery(xrange(4), create_call=_source))
ArtifactContent.all().AndReturn(MockQuery(xrange(12), create_call=_content))
moxer.ReplayAll()
sources, text = new_default_mixer().mix_random_limit_sources(1)
self.assertEquals(len(sources), 1)
moxer.VerifyAll()
def test_mix_random_limit_sources_1(self):
moxer = mox.Mox()
moxer.StubOutWithMock(ArtifactSource, "all")
moxer.StubOutWithMock(ArtifactContent, "all")
ArtifactSource.all().AndReturn(
MockQuery(xrange(4), create_call=_source))
ArtifactContent.all().AndReturn(
MockQuery(xrange(12), create_call=_content))
moxer.ReplayAll()
sources, text = new_default_mixer().mix_random_limit_sources(1)
self.assertEquals(len(sources), 1)
moxer.VerifyAll()
def delete(self, **kw):
helper = RequestHelper(self)
q = self.request.get("q", None)
if not q:
helper.error(400, "q not provided.")
return
results = {}
deleted_guids = []
results["deleted"] = deleted_guids
errors = 0
q_results = ArtifactContent.all().search(q)
infos = []
batch_size = 50
q_count = q_results.count()
batches = (q_count / batch_size) + 1
count = 0
for i in range(0, batches):
for c in q_results.fetch(batch_size, i * batch_size):
try:
logging.debug("deleting guid:%s" % c.guid)
c.delete()
count += 1
deleted_guids.append(c.guid)
if c.info is not None:
c.info.delete()
except Exception, ex:
logging.error(ex)
errors += 1
def get(self, **kw):
helper = RequestHelper(self)
q = self.request.get("q", None)
output = self.request.get("o", None)
max_results = int(self.request.get("max", -1))
if not q:
helper.error(400, "q not provided.")
return
q_results = ArtifactContent.all().search(q)
json_results = None
if output == "short":
json_results = {}
json_results["count"] = q_results.count()
elif output == "id":
json_results = {}
count = q_results.count()
if max_results > 0 and max_results < q_results.count():
count = max_results
json_results["count"] = count
ids = []
json_results["ids"] = ids
results = q_results.fetch(1000) if max_results == -1 else q_results.fetch(max_results)
for c in results:
ids.append(c.guid)
else:
json_results = []
if q_results.count():
for content in q_results.fetch(10):
info = ArtifactInfo.get_by_guid(content.guid)
json_results.append(ArtifactsHelper.artifact_to_hash(info, content))
helper.write_json(json_results)
def test_words(words, text_call):
for i in xrange(len(words)):
moxer = mox.Mox()
moxer.StubOutWithMock(ArtifactSource, "all")
moxer.StubOutWithMock(ArtifactContent, "all")
ArtifactSource.all().AndReturn(MockQuery(xrange(4), create_call=_source))
word = words[i]
source_content = _content(word)
source_text = source_content.body
ArtifactContent.all().AndReturn(MockQuery(xrange(1), create_call=lambda id: source_content))
moxer.ReplayAll()
mixer = new_default_mixer()
sources, mixed_text = mixer.mix_random_limit_sources(1)
print "('%s') '%s' -> '%s'" % (word, source_text, mixed_text)
text_call(word, source_text, mixed_text)
moxer.VerifyAll()
def test_mix_sources_1(self):
moxer = mox.Mox()
moxer.StubOutWithMock(ArtifactSource, "get_multiple_by_name")
moxer.StubOutWithMock(ArtifactContent, "all")
source_name = "source/1"
source = MockEntity(key_name=source_name, name=source_name)
def _content(id):
id_str = str(id)
return MockEntity(key_name=id_str, guid=id_str, body=generate_phrase(5), source=source, source_name=source.name)
ArtifactSource.get_multiple_by_name(source_name).AndReturn((source, ))
ArtifactContent.all().AndReturn(MockQuery(xrange(12), create_call=_content))
moxer.ReplayAll()
sources, text = new_default_mixer().mix_sources(source_name)
self.assertEquals(len(sources), 1)
moxer.VerifyAll()
def test_words(words, text_call):
for i in xrange(len(words)):
moxer = mox.Mox()
moxer.StubOutWithMock(ArtifactSource, "all")
moxer.StubOutWithMock(ArtifactContent, "all")
ArtifactSource.all().AndReturn(
MockQuery(xrange(4), create_call=_source))
word = words[i]
source_content = _content(word)
source_text = source_content.body
ArtifactContent.all().AndReturn(
MockQuery(xrange(1),
create_call=lambda id: source_content))
moxer.ReplayAll()
mixer = new_default_mixer()
sources, mixed_text = mixer.mix_random_limit_sources(1)
print "('%s') '%s' -> '%s'" % (word, source_text, mixed_text)
text_call(word, source_text, mixed_text)
moxer.VerifyAll()
def __random_content_for_source(self, source, count, minimum_results=MINIMUM_ARTIFACTS):
"""
returns:
list of ArtifactContent records
"""
if not source:
raise IllegalArgumentException("source cannot be None")
logging.debug("__random_content_for_source source:%s" % source)
content_q = ArtifactContent.all().filter("source =", source)
content_q_count = content_q.count()
if content_q_count < minimum_results:
msg = "not enough ArtifactContents found for ArtifactSource %s (%d < %d)" % \
(source.name, content_q_count, minimum_results)
raise MissingDataException(msg)
return random_query_results(content_q, count)
def __random_content_for_source(self,
source,
count,
minimum_results=MINIMUM_ARTIFACTS):
"""
returns:
list of ArtifactContent records
"""
if not source:
raise IllegalArgumentException("source cannot be None")
logging.debug("__random_content_for_source source:%s" % source)
content_q = ArtifactContent.all().filter("source =", source)
content_q_count = content_q.count()
if content_q_count < minimum_results:
msg = "not enough ArtifactContents found for ArtifactSource %s (%d < %d)" % \
(source.name, content_q_count, minimum_results)
raise MissingDataException(msg)
return random_query_results(content_q, count)
def mix_latest(self):
q = ArtifactContent.all().order("-modified")
contents = self.__random_content_from_query(q)
return self.__mix_contents(contents)
def search(cls, term, max_results=50):
# TODO: tweak result limit
return [
c for c in ArtifactContent.all().search(term).fetch(max_results)
]
def search(cls, term, max_results=50):
# TODO: tweak result limit
return [c for c in ArtifactContent.all().search(term).fetch(max_results)]
def mix_latest(self):
q = ArtifactContent.all().order("-modified")
contents = self.__random_content_from_query(q)
return self.__mix_contents(contents)
