def random_resource(self, max_length=130):
resource = None
q = UrlResource.find_latest()
if not q.count():
raise MissingDataException("no results found")
for resource in random_query_results(q, 10):
if len(resource.url) < max_length:
break
if not resource: raise MissingDataException("no sufficient resources found")
return resource
def random_resource(self, max_length=130):
resource = None
q = UrlResource.find_latest()
if not q.count():
raise MissingDataException("no results found")
for resource in random_query_results(q, 10):
if len(resource.url) < max_length:
break
if not resource:
raise MissingDataException("no sufficient resources found")
return resource
def __random_content_from_query(self, q, count=CONTENT_BATCH_SIZE, minimum_results=MINIMUM_ARTIFACTS):
"""
returns:
list of ArtifactContent records
"""
q_count = q.count()
if q_count < minimum_results:
msg = "not enough ArtifactContents found in query (%d < %d)" % \
(q_count, minimum_results)
logging.error(msg)
raise MissingDataException(msg)
return random_query_results(q, count)
def __random_content_from_query(self,
q,
count=CONTENT_BATCH_SIZE,
minimum_results=MINIMUM_ARTIFACTS):
"""
returns:
list of ArtifactContent records
"""
q_count = q.count()
if q_count < minimum_results:
msg = "not enough ArtifactContents found in query (%d < %d)" % \
(q_count, minimum_results)
logging.error(msg)
raise MissingDataException(msg)
return random_query_results(q, count)
def __random_content_for_source(self, source, count, minimum_results=MINIMUM_ARTIFACTS):
"""
returns:
list of ArtifactContent records
"""
if not source:
raise IllegalArgumentException("source cannot be None")
logging.debug("__random_content_for_source source:%s" % source)
content_q = ArtifactContent.all().filter("source =", source)
content_q_count = content_q.count()
if content_q_count < minimum_results:
msg = "not enough ArtifactContents found for ArtifactSource %s (%d < %d)" % \
(source.name, content_q_count, minimum_results)
raise MissingDataException(msg)
return random_query_results(content_q, count)
def __random_content_for_source(self,
source,
count,
minimum_results=MINIMUM_ARTIFACTS):
"""
returns:
list of ArtifactContent records
"""
if not source:
raise IllegalArgumentException("source cannot be None")
logging.debug("__random_content_for_source source:%s" % source)
content_q = ArtifactContent.all().filter("source =", source)
content_q_count = content_q.count()
if content_q_count < minimum_results:
msg = "not enough ArtifactContents found for ArtifactSource %s (%d < %d)" % \
(source.name, content_q_count, minimum_results)
raise MissingDataException(msg)
return random_query_results(content_q, count)
def mix_random_limit_sources(self, source_count, degrade=False):
"""
params:
source_count - number of sources to mix
degrade - if True, mix even if source_count sources cannot be found
returns:
((sources), mixed_content)
"""
# choose random sources
source_q = ArtifactSource.all()
q_count = source_q.count()
if (q_count < source_count):
if degrade:
logging.debug("mix_random_limit_sources requested %d sources; degrading to %d" % (source_count, q_count))
source_count = q_count
else:
raise MissingDataException("insufficient ArtifactSources found (%d < %d)" % \
(q_count, source_count))
sources = random_query_results(source_q, source_count)
return self.__random_content_for_sources(sources)
def mix_random_limit_sources(self, source_count, degrade=False):
"""
params:
source_count - number of sources to mix
degrade - if True, mix even if source_count sources cannot be found
returns:
((sources), mixed_content)
"""
# choose random sources
source_q = ArtifactSource.all()
q_count = source_q.count()
if (q_count < source_count):
if degrade:
logging.debug(
"mix_random_limit_sources requested %d sources; degrading to %d"
% (source_count, q_count))
source_count = q_count
else:
raise MissingDataException("insufficient ArtifactSources found (%d < %d)" % \
(q_count, source_count))
sources = random_query_results(source_q, source_count)
return self.__random_content_for_sources(sources)