예제 #1
0
    def handle(self, *args, **options):
        es = amcates.ES()

        print("Counting articles..", end=" ")
        sys.stdout.flush()
        narticles = es.count(query="*", filters={})
        print(narticles)

        then, now = datetime.datetime.now(), datetime.datetime.now()
        for i, article_ids in enumerate(grouper(es.query_ids(), n=GROUP_SIZE)):
            progress = (float(i * GROUP_SIZE) / float(narticles)) * 100
            print("{} of {} ({:.2f}%)".format(i * GROUP_SIZE, narticles,
                                              progress))

            articles = Article.objects.filter(
                id__in=article_ids).select_related("medium")
            article_dicts = map(get_article_dict, articles)

            for article_dict in article_dicts:
                del article_dict["sets"]
                del article_dict["hash"]

            es.bulk_update_values({a["id"]: a for a in article_dicts})

            then, now = now, datetime.datetime.now()
            print("Articles per second: ", end="")
            print(int(GROUP_SIZE / (now - then).total_seconds()))

        print("Done.")
예제 #2
0
    def handle(self, *args, **options):
        es = amcates.ES()

        print("Counting articles..", end=" ")
        sys.stdout.flush()
        narticles = es.count(query="*", filters={})
        print(narticles)

        then, now = datetime.datetime.now(), datetime.datetime.now()
        for i, article_ids in enumerate(grouper(es.query_ids(), n=GROUP_SIZE)):
            progress = (float(i * GROUP_SIZE) / float(narticles)) * 100
            print("{} of {} ({:.2f}%)".format(i*GROUP_SIZE, narticles, progress))

            articles = Article.objects.filter(id__in=article_ids).select_related("medium")
            article_dicts = map(get_article_dict, articles)

            for article_dict in article_dicts:
                del article_dict["sets"]
                del article_dict["hash"]

            es.bulk_update_values({a["id"]: a for a in article_dicts})

            then, now = now, datetime.datetime.now()
            print("Articles per second: ", end="")
            print(int(GROUP_SIZE / (now - then).total_seconds()))

        print("Done.")
예제 #3
0
    def handle(self, *args, **options):
        es = amcates.ES()

        print("Counting articles..", end=" ")
        sys.stdout.flush()
        narticles = es.count(query="*", filters={})
        print(narticles)

        then, now = datetime.datetime.now(), datetime.datetime.now()
        for i, article_ids in enumerate(grouper(es.query_ids(), n=GROUP_SIZE)):
            progress = (float(i * GROUP_SIZE) / float(narticles)) * 100
            print("{} of {} ({:.2f}%)".format(i * GROUP_SIZE, narticles,
                                              progress))

            es_articles = es.query_all(filters={"ids": article_ids},
                                       fields=HASH_FIELDS)
            es.bulk_update_values(
                {a.id: {
                    "hash": _get_hash(a.to_dict())
                }
                 for a in es_articles})

            then, now = now, datetime.datetime.now()
            print("Articles per second: ", end="")
            print(int(GROUP_SIZE / (now - then).total_seconds()))

        print("Done.")
예제 #4
0
    def handle(self, *args, **options):
        es = amcates.ES()

        print("Counting articles..", end=" ")
        sys.stdout.flush()
        narticles = es.count(query="*", filters={})
        print(narticles)

        then, now = datetime.datetime.now(), datetime.datetime.now()
        for i, article_ids in enumerate(grouper(es.query_ids(), n=GROUP_SIZE)):
            progress = (float(i * GROUP_SIZE) / float(narticles)) * 100
            print("{} of {} ({:.2f}%)".format(i*GROUP_SIZE, narticles, progress))

            es_articles = es.query_all(filters={"ids": article_ids}, fields=HASH_FIELDS)
            es.bulk_update_values({a.id: {"hash": _get_hash(a.to_dict())} for a in es_articles})

            then, now = now, datetime.datetime.now()
            print("Articles per second: ", end="")
            print(int(GROUP_SIZE / (now - then).total_seconds()))

        print("Done.")