示例#1
0
 def random_resource(self, max_length=130):
     resource = None
     q = UrlResource.find_latest()
     if not q.count():
         raise MissingDataException("no results found")
         
     for resource in random_query_results(q, 10):
         if len(resource.url) < max_length:
             break
     if not resource: raise MissingDataException("no sufficient resources found")
     return resource
示例#2
0
    def random_resource(self, max_length=130):
        resource = None
        q = UrlResource.find_latest()
        if not q.count():
            raise MissingDataException("no results found")

        for resource in random_query_results(q, 10):
            if len(resource.url) < max_length:
                break
        if not resource:
            raise MissingDataException("no sufficient resources found")
        return resource
示例#3
0
 def __random_content_from_query(self, q, count=CONTENT_BATCH_SIZE, minimum_results=MINIMUM_ARTIFACTS):
     """
     returns:
         list of ArtifactContent records
     """
     q_count = q.count()
     if q_count < minimum_results:
         msg = "not enough ArtifactContents found in query (%d < %d)" % \
             (q_count, minimum_results)
         logging.error(msg)
         raise MissingDataException(msg)
     return random_query_results(q, count)
示例#4
0
 def __random_content_from_query(self,
                                 q,
                                 count=CONTENT_BATCH_SIZE,
                                 minimum_results=MINIMUM_ARTIFACTS):
     """
     returns:
         list of ArtifactContent records
     """
     q_count = q.count()
     if q_count < minimum_results:
         msg = "not enough ArtifactContents found in query (%d < %d)" % \
             (q_count, minimum_results)
         logging.error(msg)
         raise MissingDataException(msg)
     return random_query_results(q, count)
示例#5
0
 def __random_content_for_source(self, source, count, minimum_results=MINIMUM_ARTIFACTS):
     """
     returns:
         list of ArtifactContent records
     """
     if not source:
         raise IllegalArgumentException("source cannot be None")
         
     logging.debug("__random_content_for_source source:%s" % source)
     content_q = ArtifactContent.all().filter("source =", source)
     content_q_count = content_q.count()
     if content_q_count < minimum_results:
         msg = "not enough ArtifactContents found for ArtifactSource %s (%d < %d)" % \
             (source.name, content_q_count, minimum_results)
         raise MissingDataException(msg)
     return random_query_results(content_q, count)
示例#6
0
    def __random_content_for_source(self,
                                    source,
                                    count,
                                    minimum_results=MINIMUM_ARTIFACTS):
        """
        returns:
            list of ArtifactContent records
        """
        if not source:
            raise IllegalArgumentException("source cannot be None")

        logging.debug("__random_content_for_source source:%s" % source)
        content_q = ArtifactContent.all().filter("source =", source)
        content_q_count = content_q.count()
        if content_q_count < minimum_results:
            msg = "not enough ArtifactContents found for ArtifactSource %s (%d < %d)" % \
                (source.name, content_q_count, minimum_results)
            raise MissingDataException(msg)
        return random_query_results(content_q, count)
示例#7
0
 def mix_random_limit_sources(self, source_count, degrade=False):
     """
     params:
         source_count - number of sources to mix
         degrade - if True, mix even if source_count sources cannot be found
     returns:
         ((sources), mixed_content)
     """
     # choose random sources
     source_q = ArtifactSource.all()
     q_count = source_q.count()
     if (q_count < source_count):
         if degrade:
             logging.debug("mix_random_limit_sources requested %d sources; degrading to %d" % (source_count, q_count))
             source_count = q_count
         else:
             raise MissingDataException("insufficient ArtifactSources found (%d < %d)" % \
                 (q_count, source_count))
     sources = random_query_results(source_q, source_count)
     return self.__random_content_for_sources(sources)
示例#8
0
 def mix_random_limit_sources(self, source_count, degrade=False):
     """
     params:
         source_count - number of sources to mix
         degrade - if True, mix even if source_count sources cannot be found
     returns:
         ((sources), mixed_content)
     """
     # choose random sources
     source_q = ArtifactSource.all()
     q_count = source_q.count()
     if (q_count < source_count):
         if degrade:
             logging.debug(
                 "mix_random_limit_sources requested %d sources; degrading to %d"
                 % (source_count, q_count))
             source_count = q_count
         else:
             raise MissingDataException("insufficient ArtifactSources found (%d < %d)" % \
                 (q_count, source_count))
     sources = random_query_results(source_q, source_count)
     return self.__random_content_for_sources(sources)