Example #1
0
    def index_view(self):
        # Grab parameters from URL
        page, sort_idx, sort_desc, search = self._get_list_extra_args()

        page_size = 10
        count = Shreds._get_collection().count()
        num_pages = count // page_size
        if count % page_size != 0:
            num_pages += 1

        data = Shreds._get_collection().find({}, {
            'contour': 0
        }).skip(page * page_size).limit(page_size)

        # Various URL generation helpers
        def pager_url(p):
            # Do not add page number if it is first page
            if p == 0:
                p = None

            return self._get_url('.index_view', p, sort_idx, sort_desc, search)

        return self.render(
            'admin/shreds.html',
            data=data,
            count=count,
            pager_url=pager_url,
            num_pages=num_pages,
            page=page,
        )
Example #2
0
    def index_view(self):
        # Grab parameters from URL
        page, sort_idx, sort_desc, search = self._get_list_extra_args()

        page_size = 10
        count = Shreds._get_collection().count()
        num_pages = count // page_size
        if count % page_size != 0:
            num_pages += 1

        data = Shreds._get_collection().find(
            {},
            {'contour': 0}).skip(page * page_size).limit(page_size)

        # Various URL generation helpers
        def pager_url(p):
            # Do not add page number if it is first page
            if p == 0:
                p = None

            return self._get_url('.index_view', p, sort_idx, sort_desc,
                                 search)

        return self.render('admin/shreds.html', data=data, count=count,
                           pager_url=pager_url,
                           num_pages=num_pages,
                           page=page,)
Example #3
0
def next():
    if request.method == "POST":
        # TODO: helper
        tags = set(map(unicode.lower, request.form.getlist("tags")))

        Shreds.objects(pk=request.form["_id"]).update_one(
            push__tags=ShredTags(
                user=g.user.id,
                tags=list(tags),
                recognizable_chars=request.form.get("recognizable_chars", ""),
                angle=int(request.form.get("angle", 0))),
            inc__users_count=1,
            add_to_set__users_processed=g.user.id)

        User.objects(pk=g.user.id).update_one(
            inc__processed=1, inc__tags_count=len(tags),
            add_to_set__tags=list(tags))

        session["processed"] = session.get("processed", 0) + 1

        for tag in tags:
            Tags.objects(pk=tag).update_one(
                set_on_insert__is_base=False,
                set_on_insert__created_by=g.user.id,
                set_on_insert__created_at=Tags().created_at,
                inc__usages=1,
                add_to_set__shreds=request.form["_id"],
                upsert=True)

        start = datetime.strptime(request.form["tagging_start"],
                                  '%Y-%m-%d %H:%M:%S.%f')
        end = datetime.utcnow()
        TaggingSpeed.objects.create(
            user=g.user.id,
            shred=request.form["_id"],
            tags_count=len(tags),
            msec=(end - start).total_seconds() * 1000)

    shred = get_next_shred()
    return render_template(
        "_shred.html",
        shred=shred,
        auto_tags=get_auto_tags(shred),
        all_tags=get_tags(),
        tagging_start=datetime.utcnow(),

        # TODO: move to context processor
        processed_per_session=session.get("processed", 0),
        processed_total=User.objects(id=g.user.id).first()["processed"],
        rating=list(User.objects.order_by(
            "-processed").values_list("id")).index(g.user.id) + 1
    )
Example #4
0
def load_new_batch(flt, batch):
    if app.config["S3_ENABLED"]:
        strg = S3Storage(app.config)
    else:
        strg = LocalFSStorage(app.config)

    pages_processed = 0
    shreds_created = 0
    import_took = time.time()

    out_dir = os.path.join(app.config["SPLIT_OUT_DIR"], "batch_%s" % batch)
    strg.clear(out_dir)
    Shreds.objects(batch=batch).delete()

    for src_key in strg.list(flt):
        fname = strg.get_file(src_key)
        sheet_name = os.path.splitext(os.path.basename(fname))[0]

        echo("\n\nProcessing file %s from %s" % (fname, sheet_name))
        sheet = SheetIO(fname, sheet_name, [GeometryFeatures, ColourFeatures],
                        out_dir, "png")

        pages_processed += 1

        for c in sheet.get_shreds():
            c = c._asdict()
            c["id"] = "%s:%s_%s" % (batch, c["sheet"], c["name"])
            c["usersCount"] = 0
            c["batch"] = batch
            shreds_created += 1

            del(c["simplified_contour"])
            c["contour"] = c["contour"].tolist()

            imgs = "piece_fname", "features_fname", "piece_in_context_fname"

            for k in imgs:
                if k in c:
                    res = strg.put_file(c[k])
                    c[k] = res

            try:
                Shreds.objects.create(**c)
            except bson.errors.InvalidDocument:
                echo(c)
                raise

    Batches(
        _id=batch,
        name=batch,
        shreds_created=shreds_created,
        pages_processed=pages_processed,
        import_took=int((time.time() - import_took) * 1000)
    ).save()

    Shreds.ensure_index(["name", "sheet", "batch"])
    Shreds.ensure_index(["users_processed", "users_count", "batch"])
    Shreds.ensure_index(["users_skipped", "users_count", "batch"])
Example #5
0
def pages():
    if request.method == "POST":
        shreds = set(request.form.getlist("shreds"))
        page_name = request.form.get("page_name")
        page_id = request.form.get("page_id")

        if page_id:
            page = Pages.objects.get(pk=page_id)
        else:
            page, _ = Pages.objects.get_or_create(
                created_by=g.user.id, name=page_name)

        page.update(add_to_set__shreds=shreds)

        for shred in Shreds.objects(id__in=shreds):
            tags = shred.get_user_tags(g.user)
            if tags is not None:
                tags.pages = list(set(tags.pages + [page]))
            # TODO: else 404?

            shred.save()

    pages = Pages.objects(created_by=g.user.id)

    return render_template(
        "_pages.html",
        pages=pages)
Example #6
0
    def index_view(self):
        # Grab parameters from URL
        page, sort_idx, sort_desc, search = self._get_list_extra_args()

        page_size = 10
        count = User._get_collection().count()
        num_pages = count // page_size
        if count % page_size != 0:
            num_pages += 1

        # data = User._get_collection().find({}, {'password': 0}).skip(page*page_size).limit(page_size)
        data = User._get_collection().find({}, {'password': 0})
        result = []
        for d in data:
            temp_dict = {}
            temp_dict['username'] = d['username']
            temp_dict['last_login'] = d.get('last_login', None)
            temp_dict['used_tags'] = []
            temp_dict['tags_count'] = 0
            temp_dict['shreds_count'] = 0
            temp_dict['skipped_count'] = 0

            shreds = Shreds._get_collection().find({"tags.user": str(d["_id"])})
            for s in shreds:
                for tag in s['tags']:
                    if tag['user'] == str(d["_id"]):
                        if tag['tags'] == 'skipped':
                            temp_dict['skipped_count'] += 1
                        else:
                            temp_dict['shreds_count'] += 1
                            temp_dict['used_tags'] += tag['tags']
                            temp_dict['tags_count'] += len(tag['tags'])

            temp_dict['used_tags'] = list(set(temp_dict['used_tags']))
            result.append(temp_dict)
        # Various URL generation helpers
        def pager_url(p):
            # Do not add page number if it is first page
            if p == 0:
                p = None

            return self._get_url('.index_view', p, sort_idx, sort_desc,
                                 search)

        def sort_url(column, invert=False):
            desc = None

            if invert and not sort_desc:
                desc = 1

            return self._get_url('.index_view', page, column, desc,
                                 search)

        return self.render('admin/users.html', data=result, count=count,
                               pager_url=pager_url,
                               num_pages=num_pages,
                               page=page,
                               list_columns=self.column_labels,
                               sort_url=sort_url,
                               get_value=self.get_list_value)
Example #7
0
def get_next_shred():
    shred = Shreds\
        .objects(users_processed__ne=g.user.id, users_skipped__ne=g.user.id,
                 users_count__lte=app.config["USERS_PER_SHRED"])\
        .order_by("batch", "users_count").first()

    if shred:
        return shred

    shred = Shreds\
        .objects(users_skipped=g.user.id,
                 users_count__lte=app.config["USERS_PER_SHRED"])\
        .order_by("batch", "users_count").first()

    if shred:
        Shreds.objects(id=shred.id).update_one(pull__users_skipped=g.user.id)

    return shred
Example #8
0
    def test_skipping(self):
        self.create_user_and_login("user")
        user = User.objects.get(username="******")
        self.assertEqual(user.skipped, 0)
        self.assertEqual(user.processed, 0)

        self.client.post(url_for("fixtures.create_shreds"))

        res = self.client.get(url_for("next"))
        self.assert200(res)
        body = res.get_data(as_text=True)

        current_shred_id = first_shred_id = self.parse_shred_id(body)

        for i in xrange(9):
            res = self.client.post(url_for("skip"),
                                   data={"_id": current_shred_id},
                                   follow_redirects=True)

            body = res.get_data(as_text=True)
            self.assert200(res)

            current_shred_id = self.parse_shred_id(body)
            self.assertNotEqual(current_shred_id, first_shred_id)

        self.assertEqual(
            len(Shreds.objects(id=first_shred_id).first().users_skipped), 1)

        res = self.client.post(url_for("skip"),
                               data={"_id": current_shred_id},
                               follow_redirects=True)

        body = res.get_data(as_text=True)
        self.assert200(res)

        current_shred_id = self.parse_shred_id(body)
        self.assertEqual(current_shred_id, first_shred_id)

        self.assertEqual(
            len(Shreds.objects(id=first_shred_id).first().users_skipped), 0)

        user.reload()
        self.assertEqual(user.skipped, 10)
        self.assertEqual(user.processed, 0)
Example #9
0
def fetch_normalized_shreds_tags(repeats):
    """
    Returns dictionary where keys are shreds ids and values are sets of
    filtered normalized tags.
    """
    shreds = Shreds.objects().only('id', 'tags.tags')[:SHREDS_CAP]
    shreds_tags = {}
    for s in shreds:
        tags = s.get_repeated_tags(repeats)
        if tags:
            shreds_tags[s.id] = tags
    return shreds_tags
Example #10
0
def skip():
    Shreds.objects(pk=request.form["_id"]).update_one(
        add_to_set__users_skipped=g.user.id)
    User.objects(pk=g.user.id).update_one(inc__skipped=1)

    return redirect(url_for("next"))
Example #11
0
app = Flask(__name__)
app.config.from_object('settings')

try:
    app.config.from_object('local_settings')
except ImportError:
    pass

assets_init(app)
admin_init(app)

db = MongoEngine(app)

init_social_login(app, db)

shreds = Shreds._get_collection()
users = User._get_collection()


def get_next_shred():
    shred = shreds.find_one(
        {"$query": {"usersProcessed": {"$ne": str(g.user.id)},
                    "usersSkipped": {"$ne": str(g.user.id)},
                    "usersCount": {"$lte": app.config["USERS_PER_SHRED"]}
                    }}, sort=[("batch", 1),
                              ("usersCount", 1)])

    if shred:
        return shred

    shred = shreds.find_one(
Example #12
0
    def index_view(self):
        # Grab parameters from URL
        page, sort_idx, sort_desc, search = self._get_list_extra_args()

        page_size = 10
        count = User._get_collection().count()
        num_pages = count // page_size
        if count % page_size != 0:
            num_pages += 1

        # data = User._get_collection().find({}, {'password': 0}).skip(page*page_size).limit(page_size)
        data = User._get_collection().find({}, {'password': 0})
        result = []
        for d in data:
            temp_dict = {}
            temp_dict['username'] = d['username']
            temp_dict['last_login'] = d.get('last_login', None)
            temp_dict['used_tags'] = []
            temp_dict['tags_count'] = 0
            temp_dict['shreds_count'] = 0
            temp_dict['skipped_count'] = 0

            shreds = Shreds._get_collection().find(
                {"tags.user": str(d["_id"])})
            for s in shreds:
                for tag in s['tags']:
                    if tag['user'] == str(d["_id"]):
                        if tag['tags'] == 'skipped':
                            temp_dict['skipped_count'] += 1
                        else:
                            temp_dict['shreds_count'] += 1
                            temp_dict['used_tags'] += tag['tags']
                            temp_dict['tags_count'] += len(tag['tags'])

            temp_dict['used_tags'] = list(set(temp_dict['used_tags']))
            result.append(temp_dict)
        # Various URL generation helpers
        def pager_url(p):
            # Do not add page number if it is first page
            if p == 0:
                p = None

            return self._get_url('.index_view', p, sort_idx, sort_desc, search)

        def sort_url(column, invert=False):
            desc = None

            if invert and not sort_desc:
                desc = 1

            return self._get_url('.index_view', page, column, desc, search)

        return self.render('admin/users.html',
                           data=result,
                           count=count,
                           pager_url=pager_url,
                           num_pages=num_pages,
                           page=page,
                           list_columns=self.column_labels,
                           sort_url=sort_url,
                           get_value=self.get_list_value)
Example #13
0
app = Flask(__name__)
app.config.from_object('settings')

try:
    app.config.from_object('local_settings')
except ImportError:
    pass

assets_init(app)
admin_init(app)

db = MongoEngine(app)

init_social_login(app, db)

shreds = Shreds._get_collection()
users = User._get_collection()


def get_next_shred():
    shred = shreds.find_one(
        {
            "$query": {
                "usersProcessed": {
                    "$ne": str(g.user.id)
                },
                "usersSkipped": {
                    "$ne": str(g.user.id)
                },
                "usersCount": {
                    "$lte": app.config["USERS_PER_SHRED"]