def index_view(self):
# Grab parameters from URL
page, sort_idx, sort_desc, search = self._get_list_extra_args()
page_size = 10
count = Shreds._get_collection().count()
num_pages = count // page_size
if count % page_size != 0:
num_pages += 1
data = Shreds._get_collection().find({}, {
'contour': 0
}).skip(page * page_size).limit(page_size)
# Various URL generation helpers
def pager_url(p):
# Do not add page number if it is first page
if p == 0:
p = None
return self._get_url('.index_view', p, sort_idx, sort_desc, search)
return self.render(
'admin/shreds.html',
data=data,
count=count,
pager_url=pager_url,
num_pages=num_pages,
page=page,
)
def index_view(self):
# Grab parameters from URL
page, sort_idx, sort_desc, search = self._get_list_extra_args()
page_size = 10
count = Shreds._get_collection().count()
num_pages = count // page_size
if count % page_size != 0:
num_pages += 1
data = Shreds._get_collection().find(
{},
{'contour': 0}).skip(page * page_size).limit(page_size)
# Various URL generation helpers
def pager_url(p):
# Do not add page number if it is first page
if p == 0:
p = None
return self._get_url('.index_view', p, sort_idx, sort_desc,
search)
return self.render('admin/shreds.html', data=data, count=count,
pager_url=pager_url,
num_pages=num_pages,
page=page,)
def next():
if request.method == "POST":
# TODO: helper
tags = set(map(unicode.lower, request.form.getlist("tags")))
Shreds.objects(pk=request.form["_id"]).update_one(
push__tags=ShredTags(
user=g.user.id,
tags=list(tags),
recognizable_chars=request.form.get("recognizable_chars", ""),
angle=int(request.form.get("angle", 0))),
inc__users_count=1,
add_to_set__users_processed=g.user.id)
User.objects(pk=g.user.id).update_one(
inc__processed=1, inc__tags_count=len(tags),
add_to_set__tags=list(tags))
session["processed"] = session.get("processed", 0) + 1
for tag in tags:
Tags.objects(pk=tag).update_one(
set_on_insert__is_base=False,
set_on_insert__created_by=g.user.id,
set_on_insert__created_at=Tags().created_at,
inc__usages=1,
add_to_set__shreds=request.form["_id"],
upsert=True)
start = datetime.strptime(request.form["tagging_start"],
'%Y-%m-%d %H:%M:%S.%f')
end = datetime.utcnow()
TaggingSpeed.objects.create(
user=g.user.id,
shred=request.form["_id"],
tags_count=len(tags),
msec=(end - start).total_seconds() * 1000)
shred = get_next_shred()
return render_template(
"_shred.html",
shred=shred,
auto_tags=get_auto_tags(shred),
all_tags=get_tags(),
tagging_start=datetime.utcnow(),
# TODO: move to context processor
processed_per_session=session.get("processed", 0),
processed_total=User.objects(id=g.user.id).first()["processed"],
rating=list(User.objects.order_by(
"-processed").values_list("id")).index(g.user.id) + 1
)
def load_new_batch(flt, batch):
if app.config["S3_ENABLED"]:
strg = S3Storage(app.config)
else:
strg = LocalFSStorage(app.config)
pages_processed = 0
shreds_created = 0
import_took = time.time()
out_dir = os.path.join(app.config["SPLIT_OUT_DIR"], "batch_%s" % batch)
strg.clear(out_dir)
Shreds.objects(batch=batch).delete()
for src_key in strg.list(flt):
fname = strg.get_file(src_key)
sheet_name = os.path.splitext(os.path.basename(fname))[0]
echo("\n\nProcessing file %s from %s" % (fname, sheet_name))
sheet = SheetIO(fname, sheet_name, [GeometryFeatures, ColourFeatures],
out_dir, "png")
pages_processed += 1
for c in sheet.get_shreds():
c = c._asdict()
c["id"] = "%s:%s_%s" % (batch, c["sheet"], c["name"])
c["usersCount"] = 0
c["batch"] = batch
shreds_created += 1
del(c["simplified_contour"])
c["contour"] = c["contour"].tolist()
imgs = "piece_fname", "features_fname", "piece_in_context_fname"
for k in imgs:
if k in c:
res = strg.put_file(c[k])
c[k] = res
try:
Shreds.objects.create(**c)
except bson.errors.InvalidDocument:
echo(c)
raise
Batches(
_id=batch,
name=batch,
shreds_created=shreds_created,
pages_processed=pages_processed,
import_took=int((time.time() - import_took) * 1000)
).save()
Shreds.ensure_index(["name", "sheet", "batch"])
Shreds.ensure_index(["users_processed", "users_count", "batch"])
Shreds.ensure_index(["users_skipped", "users_count", "batch"])
def pages():
if request.method == "POST":
shreds = set(request.form.getlist("shreds"))
page_name = request.form.get("page_name")
page_id = request.form.get("page_id")
if page_id:
page = Pages.objects.get(pk=page_id)
else:
page, _ = Pages.objects.get_or_create(
created_by=g.user.id, name=page_name)
page.update(add_to_set__shreds=shreds)
for shred in Shreds.objects(id__in=shreds):
tags = shred.get_user_tags(g.user)
if tags is not None:
tags.pages = list(set(tags.pages + [page]))
# TODO: else 404?
shred.save()
pages = Pages.objects(created_by=g.user.id)
return render_template(
"_pages.html",
pages=pages)
def index_view(self):
# Grab parameters from URL
page, sort_idx, sort_desc, search = self._get_list_extra_args()
page_size = 10
count = User._get_collection().count()
num_pages = count // page_size
if count % page_size != 0:
num_pages += 1
# data = User._get_collection().find({}, {'password': 0}).skip(page*page_size).limit(page_size)
data = User._get_collection().find({}, {'password': 0})
result = []
for d in data:
temp_dict = {}
temp_dict['username'] = d['username']
temp_dict['last_login'] = d.get('last_login', None)
temp_dict['used_tags'] = []
temp_dict['tags_count'] = 0
temp_dict['shreds_count'] = 0
temp_dict['skipped_count'] = 0
shreds = Shreds._get_collection().find({"tags.user": str(d["_id"])})
for s in shreds:
for tag in s['tags']:
if tag['user'] == str(d["_id"]):
if tag['tags'] == 'skipped':
temp_dict['skipped_count'] += 1
else:
temp_dict['shreds_count'] += 1
temp_dict['used_tags'] += tag['tags']
temp_dict['tags_count'] += len(tag['tags'])
temp_dict['used_tags'] = list(set(temp_dict['used_tags']))
result.append(temp_dict)
# Various URL generation helpers
def pager_url(p):
# Do not add page number if it is first page
if p == 0:
p = None
return self._get_url('.index_view', p, sort_idx, sort_desc,
search)
def sort_url(column, invert=False):
desc = None
if invert and not sort_desc:
desc = 1
return self._get_url('.index_view', page, column, desc,
search)
return self.render('admin/users.html', data=result, count=count,
pager_url=pager_url,
num_pages=num_pages,
page=page,
list_columns=self.column_labels,
sort_url=sort_url,
get_value=self.get_list_value)
def get_next_shred():
shred = Shreds\
.objects(users_processed__ne=g.user.id, users_skipped__ne=g.user.id,
users_count__lte=app.config["USERS_PER_SHRED"])\
.order_by("batch", "users_count").first()
if shred:
return shred
shred = Shreds\
.objects(users_skipped=g.user.id,
users_count__lte=app.config["USERS_PER_SHRED"])\
.order_by("batch", "users_count").first()
if shred:
Shreds.objects(id=shred.id).update_one(pull__users_skipped=g.user.id)
return shred
def test_skipping(self):
self.create_user_and_login("user")
user = User.objects.get(username="******")
self.assertEqual(user.skipped, 0)
self.assertEqual(user.processed, 0)
self.client.post(url_for("fixtures.create_shreds"))
res = self.client.get(url_for("next"))
self.assert200(res)
body = res.get_data(as_text=True)
current_shred_id = first_shred_id = self.parse_shred_id(body)
for i in xrange(9):
res = self.client.post(url_for("skip"),
data={"_id": current_shred_id},
follow_redirects=True)
body = res.get_data(as_text=True)
self.assert200(res)
current_shred_id = self.parse_shred_id(body)
self.assertNotEqual(current_shred_id, first_shred_id)
self.assertEqual(
len(Shreds.objects(id=first_shred_id).first().users_skipped), 1)
res = self.client.post(url_for("skip"),
data={"_id": current_shred_id},
follow_redirects=True)
body = res.get_data(as_text=True)
self.assert200(res)
current_shred_id = self.parse_shred_id(body)
self.assertEqual(current_shred_id, first_shred_id)
self.assertEqual(
len(Shreds.objects(id=first_shred_id).first().users_skipped), 0)
user.reload()
self.assertEqual(user.skipped, 10)
self.assertEqual(user.processed, 0)
def fetch_normalized_shreds_tags(repeats):
"""
Returns dictionary where keys are shreds ids and values are sets of
filtered normalized tags.
"""
shreds = Shreds.objects().only('id', 'tags.tags')[:SHREDS_CAP]
shreds_tags = {}
for s in shreds:
tags = s.get_repeated_tags(repeats)
if tags:
shreds_tags[s.id] = tags
return shreds_tags
def skip():
Shreds.objects(pk=request.form["_id"]).update_one(
add_to_set__users_skipped=g.user.id)
User.objects(pk=g.user.id).update_one(inc__skipped=1)
return redirect(url_for("next"))
app = Flask(__name__)
app.config.from_object('settings')
try:
app.config.from_object('local_settings')
except ImportError:
pass
assets_init(app)
admin_init(app)
db = MongoEngine(app)
init_social_login(app, db)
shreds = Shreds._get_collection()
users = User._get_collection()
def get_next_shred():
shred = shreds.find_one(
{"$query": {"usersProcessed": {"$ne": str(g.user.id)},
"usersSkipped": {"$ne": str(g.user.id)},
"usersCount": {"$lte": app.config["USERS_PER_SHRED"]}
}}, sort=[("batch", 1),
("usersCount", 1)])
if shred:
return shred
shred = shreds.find_one(
def index_view(self):
# Grab parameters from URL
page, sort_idx, sort_desc, search = self._get_list_extra_args()
page_size = 10
count = User._get_collection().count()
num_pages = count // page_size
if count % page_size != 0:
num_pages += 1
# data = User._get_collection().find({}, {'password': 0}).skip(page*page_size).limit(page_size)
data = User._get_collection().find({}, {'password': 0})
result = []
for d in data:
temp_dict = {}
temp_dict['username'] = d['username']
temp_dict['last_login'] = d.get('last_login', None)
temp_dict['used_tags'] = []
temp_dict['tags_count'] = 0
temp_dict['shreds_count'] = 0
temp_dict['skipped_count'] = 0
shreds = Shreds._get_collection().find(
{"tags.user": str(d["_id"])})
for s in shreds:
for tag in s['tags']:
if tag['user'] == str(d["_id"]):
if tag['tags'] == 'skipped':
temp_dict['skipped_count'] += 1
else:
temp_dict['shreds_count'] += 1
temp_dict['used_tags'] += tag['tags']
temp_dict['tags_count'] += len(tag['tags'])
temp_dict['used_tags'] = list(set(temp_dict['used_tags']))
result.append(temp_dict)
# Various URL generation helpers
def pager_url(p):
# Do not add page number if it is first page
if p == 0:
p = None
return self._get_url('.index_view', p, sort_idx, sort_desc, search)
def sort_url(column, invert=False):
desc = None
if invert and not sort_desc:
desc = 1
return self._get_url('.index_view', page, column, desc, search)
return self.render('admin/users.html',
data=result,
count=count,
pager_url=pager_url,
num_pages=num_pages,
page=page,
list_columns=self.column_labels,
sort_url=sort_url,
get_value=self.get_list_value)
app = Flask(__name__)
app.config.from_object('settings')
try:
app.config.from_object('local_settings')
except ImportError:
pass
assets_init(app)
admin_init(app)
db = MongoEngine(app)
init_social_login(app, db)
shreds = Shreds._get_collection()
users = User._get_collection()
def get_next_shred():
shred = shreds.find_one(
{
"$query": {
"usersProcessed": {
"$ne": str(g.user.id)
},
"usersSkipped": {
"$ne": str(g.user.id)
},
"usersCount": {
"$lte": app.config["USERS_PER_SHRED"]