Example #1
0
def format_note_object_for_client(note):
    """
    Returns an object that represents note in the format expected by the reader client,
    matching the format of links, which are currently handled together.
    """
    anchor_oref = Ref(note.ref).padded_ref()
    ownerData   = public_user_data(note.owner)

    com = {
        "category":        "Notes",
        "type":            "note",
        "owner":           note.owner,
        "_id":             str(note._id),
        "anchorRef":       note.ref,
        "anchorVerse":     anchor_oref.sections[-1],
        "anchorText":      getattr(note, "anchorText", ""),
        "public":          getattr(note, "public", False),
        "commentator":     user_link(note.owner),
        "text":            note.text,
        "title":           getattr(note, "title", ""),
        "ownerName":       ownerData["name"],
        "ownerProfileUrl": ownerData["profileUrl"],
        "ownerImageUrl":   ownerData["imageUrl"],
    }
    return com
Example #2
0
def index_sheet(index_name, id):
    """
    Index source sheet with 'id'.
    """

    sheet = db.sheets.find_one({"id": id})
    if not sheet: return False

    pud = public_user_data(sheet["owner"])
    doc = {
        "title": sheet["title"],
        "content": make_sheet_text(sheet, pud),
        "owner_id": sheet["owner"],
        "owner_name": pud["name"],
        "owner_image": pud["imageUrl"],
        "profile_url": pud["profileUrl"],
        "version": "Source Sheet by " + user_link(sheet["owner"]),
        "tags": ",".join(sheet.get("tags",[])),
        "sheetId": id,
    }
    try:
        es.index(index_name, 'sheet', doc, id)
        global doc_count
        doc_count += 1
        return True
    except Exception, e:
        print "Error indexing sheet %d" % id
        print e
        return False
Example #3
0
def format_note_object_for_client(note):
    """
    Returns an object that represents note in the format expected by the reader client,
    matching the format of links, which are currently handled together.
    """
    anchor_oref = Ref(note.ref).padded_ref()
    ownerData = public_user_data(note.owner)

    com = {
        "category": "Notes",
        "type": "note",
        "owner": note.owner,
        "_id": str(note._id),
        "anchorRef": note.ref,
        "anchorVerse": anchor_oref.sections[-1],
        "anchorText": getattr(note, "anchorText", ""),
        "public": getattr(note, "public", False),
        "commentator": user_link(note.owner),
        "text": note.text,
        "title": getattr(note, "title", ""),
        "ownerName": ownerData["name"],
        "ownerProfileUrl": ownerData["profileUrl"],
        "ownerImageUrl": ownerData["imageUrl"],
    }
    return com
def index_sheet(index_name, id):
    """
    Index source sheet with 'id'.
    """

    sheet = db.sheets.find_one({"id": id})
    if not sheet: return False

    pud = public_user_data(sheet["owner"])
    try:
        doc = {
            "title": strip_tags(sheet["title"]),
            "content": make_sheet_text(sheet, pud),
            "owner_id": sheet["owner"],
            "owner_name": pud["name"],
            "owner_image": pud["imageUrl"],
            "profile_url": pud["profileUrl"],
            "version": "Source Sheet by " + user_link(sheet["owner"]),
            "tags": sheet.get("tags", []),
            "sheetId": id,
            "summary": sheet.get("summary", None),
            "group": sheet.get("group", None),
            "datePublished": sheet.get("datePublished", None),
            "dateCreated": sheet.get("dateCreated", None),
            "dateModified": sheet.get("dateCreated", None),
            "views": sheet.get("views", 0)
        }
        es_client.create(index=index_name, doc_type='sheet', id=id, body=doc)
        global doc_count
        doc_count += 1
        return True
    except Exception, e:
        print "Error indexing sheet %d" % id
        print e
        return False
Example #5
0
def index_sheet(index_name, id):
    """
    Index source sheet with 'id'.
    """

    sheet = db.sheets.find_one({"id": id})
    if not sheet: return False

    pud = public_user_data(sheet["owner"])
    doc = {
        "title": sheet["title"],
        "content": make_sheet_text(sheet, pud),
        "owner_id": sheet["owner"],
        "owner_name": pud["name"],
        "owner_image": pud["imageUrl"],
        "profile_url": pud["profileUrl"],
        "version": "Source Sheet by " + user_link(sheet["owner"]),
        "tags": ",".join(sheet.get("tags",[])),
        "sheetId": id,
    }
    try:
        es.index(index_name, 'sheet', doc, id)
        global doc_count
        doc_count += 1
    except Exception, e:
        print "Error indexing sheet %d" % id
        print e
Example #6
0
def annotate_user_links(sources):
	"""
	Search a sheet for any addedBy fields (containg a UID) and add corresponding user links.
	"""
	for source in sources:
		if "addedBy" in source:
			source["userLink"] = user_link(source["addedBy"])
	return sources
Example #7
0
def annotate_user_links(sources):
    """
	Search a sheet for any addedBy fields (containg a UID) and add corresponding user links.
	"""
    for source in sources:
        if "addedBy" in source:
            source["userLink"] = user_link(source["addedBy"])
    return sources
Example #8
0
def get_sheets_for_ref(tref, pad=True, context=1):
	"""
	Returns a list of sheets that include ref,
	formating as need for the Client Sidebar.
	"""
	oref = model.Ref(tref)
	if pad:
		oref = oref.padded_ref()
	if context:
		oref = oref.context_ref(context)

	ref_re = oref.regex()

	results = []

	regex_list = oref.regex(as_list=True)
	ref_clauses = [{"sources.ref": {"$regex": r}} for r in regex_list]
	sheets = db.sheets.find({"$or": ref_clauses, "status": "public"},
		{"id": 1, "title": 1, "owner": 1, "sources.ref": 1, "views": 1}).sort([["views", -1]])
	for sheet in sheets:
		matched_refs = []
		if "sources" in sheet:
			for source in sheet["sources"]:
				if "ref" in source:
					matched_refs.append(source["ref"])
		matched_refs = [r for r in matched_refs if regex.match(ref_re, r)]
		for match in matched_refs:
			try:
				match = model.Ref(match)
			except InputError:
				continue
			ownerData = public_user_data(sheet["owner"])
			com = {
				"category":        "Sheets",
				"type":            "sheet",
				"owner":           sheet["owner"],
				"_id":             str(sheet["_id"]),
				"anchorRef":       match.normal(),
				"anchorVerse":     match.sections[-1] if len(match.sections) else 1,
				"public":          True,
				"commentator":     user_link(sheet["owner"]), # legacy, used in S1
				"text":            "<a class='sheetLink' href='/sheets/%d'>%s</a>" % (sheet["id"], strip_tags(sheet["title"])), # legacy, used in S1
				"title":           strip_tags(sheet["title"]),
				"sheetUrl":        "/sheets/" + str(sheet["id"]),
				"ownerName":       ownerData["name"],
				"ownerProfileUrl": ownerData["profileUrl"],
				"ownerImageUrl":   ownerData["imageUrl"],
				"views":           sheet["views"]
			}

			results.append(com)

	return results
Example #9
0
def get_sheets_for_ref(tref, pad=True, context=1):
	"""
	Returns a list of sheets that include ref,
	formating as need for the Client Sidebar.
	"""
	oref = model.Ref(tref)
	if pad:
		oref = oref.padded_ref()
	if context:
		oref = oref.context_ref(context)

	ref_re = oref.regex()

	results = []

	regex_list = oref.regex(as_list=True)
	ref_clauses = [{"sources.ref": {"$regex": r}} for r in regex_list]
	sheets = db.sheets.find({"$or": ref_clauses, "status": "public"},
		{"id": 1, "title": 1, "owner": 1, "sources.ref": 1, "views": 1}).sort([["views", -1]])
	for sheet in sheets:
		matched_refs = []
		if "sources" in sheet:
			for source in sheet["sources"]:
				if "ref" in source:
					matched_refs.append(source["ref"])
		matched_refs = [r for r in matched_refs if regex.match(ref_re, r)]
		for match in matched_refs:
			try:
				match = model.Ref(match)
			except InputError:
				continue
			ownerData = public_user_data(sheet["owner"])
			com = {
				"category":        "Sheets",
				"type":            "sheet",
				"owner":           sheet["owner"],
				"_id":             str(sheet["_id"]),
				"anchorRef":       match.normal(),
				"anchorVerse":     match.sections[-1] if len(match.sections) else 1,
				"public":          True,
				"commentator":     user_link(sheet["owner"]), # legacy, used in S1
				"text":            "<a class='sheetLink' href='/sheets/%d'>%s</a>" % (sheet["id"], strip_tags(sheet["title"])), # legacy, used in S1
				"title":           strip_tags(sheet["title"]),
				"sheetUrl":        "/sheets/" + str(sheet["id"]),
				"ownerName":       ownerData["name"],
				"ownerProfileUrl": ownerData["profileUrl"],
				"ownerImageUrl":   ownerData["imageUrl"],
				"views":           sheet["views"]
			}

			results.append(com)

	return results
Example #10
0
def get_reviews(tref, lang, version):
	"""
	Returns a list of reviews pertaining to ref/lang/version
	"""
	reviews = []
	tref = model.Ref(tref).normal()
	refRe = '^%s$|^%s:' % (tref, tref)
	cursor = db.history.find({"ref": {"$regex": refRe}, "language": lang, "version": version, "rev_type": "review"}).sort([["date", -1]])
	for r in cursor:
		r["_id"] = str(r["_id"])
		r["userLink"] = user_link(r["user"])
		reviews.append(r)

	return reviews
Example #11
0
def index_sheet(index_name, id):
    """
    Index source sheet with 'id'.
    """

    sheet = db.sheets.find_one({"id": id})
    if not sheet: return False

    pud = public_user_data(sheet["owner"])
    tag_terms_simple = make_sheet_tags(sheet)
    tags = [t["en"] for t in tag_terms_simple]
    topics = []
    for t in sheet.get('topics', []):
        topic_obj = Topic.init(t['slug'])
        if not topic_obj:
            continue
        topics += [topic_obj]
    collections = CollectionSet({"sheets": id, "listed": True})
    collection_names = [c.name for c in collections]
    try:
        doc = {
            "title": strip_tags(sheet["title"]),
            "content": make_sheet_text(sheet, pud),
            "owner_id": sheet["owner"],
            "owner_name": pud["name"],
            "owner_image": pud["imageUrl"],
            "profile_url": pud["profileUrl"],
            "version": "Source Sheet by " + user_link(sheet["owner"]),
            "tags": tags,
            "topic_slugs": [topic_obj.slug for topic_obj in topics],
            "topics_en": [topic_obj.get_primary_title('en') for topic_obj in topics],
            "topics_he": [topic_obj.get_primary_title('he') for topic_obj in topics],
            "sheetId": id,
            "summary": sheet.get("summary", None),
            "collections": collection_names,
            "datePublished": sheet.get("datePublished", None),
            "dateCreated": sheet.get("dateCreated", None),
            "dateModified": sheet.get("dateModified", None),
            "views": sheet.get("views", 0)
        }
        es_client.create(index=index_name, doc_type='sheet', id=id, body=doc)
        global doc_count
        doc_count += 1
        return True
    except Exception as e:
        print("Error indexing sheet %d" % id)
        print(e)
        return False
Example #12
0
def index_sheet(index_name, id):
    """
    Index source sheet with 'id'.
    """

    sheet = db.sheets.find_one({"id": id})
    if not sheet: return False

    pud = public_user_data(sheet["owner"])
    tag_terms_simple = make_sheet_tags(sheet)
    tags = [t["en"] for t in tag_terms_simple]
    tags_he_and_en = [
        u"{}|||{}".format(t["en"], t["he"]) for t in tag_terms_simple
    ]
    try:
        doc = {
            "title": strip_tags(sheet["title"]),
            "content": make_sheet_text(sheet, pud),
            "owner_id": sheet["owner"],
            "owner_name": pud["name"],
            "owner_image": pud["imageUrl"],
            "profile_url": pud["profileUrl"],
            "version": "Source Sheet by " + user_link(sheet["owner"]),
            "tags": tags,
            "tags_he_and_en": tags_he_and_en,
            "sheetId": id,
            "summary": sheet.get("summary", None),
            "group": sheet.get("group", None),
            "datePublished": sheet.get("datePublished", None),
            "dateCreated": sheet.get("dateCreated", None),
            "dateModified": sheet.get("dateCreated", None),
            "views": sheet.get("views", 0)
        }
        es_client.create(index=index_name, doc_type='sheet', id=id, body=doc)
        global doc_count
        doc_count += 1
        return True
    except Exception, e:
        print "Error indexing sheet %d" % id
        print e
        return False
Example #13
0
def get_reviews(tref, lang, version):
    """
	Returns a list of reviews pertaining to ref/lang/version
	"""
    reviews = []
    tref = model.Ref(tref).normal()
    refRe = '^%s$|^%s:' % (tref, tref)
    cursor = db.history.find({
        "ref": {
            "$regex": refRe
        },
        "language": lang,
        "version": version,
        "rev_type": "review"
    }).sort([["date", -1]])
    for r in cursor:
        r["_id"] = str(r["_id"])
        r["userLink"] = user_link(r["user"])
        reviews.append(r)

    return reviews
Example #14
0
def index_sheet(id):
    """
    Index source sheet with 'id'.
    """

    sheet = db.sheets.find_one({"id": id})
    if not sheet: return False

    doc = {
        "title": sheet["title"],
        "content": make_sheet_text(sheet),
        "version": "Source Sheet by " + user_link(sheet["owner"]),
        "tags": ",".join(sheet.get("tags", [])),
        "sheetId": id,
    }
    try:
        es.index('sefaria', 'sheet', doc, id)
        global doc_count
        doc_count += 1
    except Exception, e:
        print "Error indexing sheet %d" % id
        print e
Example #15
0
def get_sheets_for_ref(tref, uid=None):
    """
	Returns a list of sheets that include ref,
	formating as need for the Client Sidebar.
	If `uid` is present return user sheets, otherwise return public sheets. 
	"""
    oref = model.Ref(tref)
    # perform initial search with context to catch ranges that include a segment ref
    regex_list = oref.context_ref().regex(as_list=True)
    ref_clauses = [{"sources.ref": {"$regex": r}} for r in regex_list]
    query = {"$or": ref_clauses}
    if uid:
        query["owner"] = uid
    else:
        query["status"] = "public"
    sheets = db.sheets.find(
        query, {
            "id": 1,
            "title": 1,
            "owner": 1,
            "sources.ref": 1,
            "views": 1,
            "tags": 1,
            "status": 1
        }).sort([["views", -1]])

    results = []
    for sheet in sheets:
        matched_refs = []
        for source in sheet.get("sources", []):
            if "ref" in source:
                matched_refs.append(source["ref"])
        for match in matched_refs:
            try:
                match = model.Ref(match)
                if not oref.overlaps(match):
                    continue
            except InputError:
                continue
            ownerData = public_user_data(sheet["owner"])
            sheet_data = {
                "owner":
                sheet["owner"],
                "_id":
                str(sheet["_id"]),
                "anchorRef":
                match.normal(),
                "anchorVerse":
                match.sections[-1] if len(match.sections) else 1,
                "public":
                sheet["status"] == "public",
                "text":
                "<a class='sheetLink' href='/sheets/%d'>%s</a>" %
                (sheet["id"], strip_tags(
                    sheet["title"])),  # legacy, used in S1
                "title":
                strip_tags(sheet["title"]),
                "sheetUrl":
                "/sheets/" + str(sheet["id"]),
                "ownerName":
                ownerData["name"],
                "ownerProfileUrl":
                ownerData["profileUrl"],
                "ownerImageUrl":
                ownerData["imageUrl"],
                "status":
                sheet["status"],
                "views":
                sheet["views"],
                "tags":
                sheet.get("tags", []),
                "commentator":
                user_link(sheet["owner"]),  # legacy, used in S1
                "category":
                "Sheets",  # ditto
                "type":
                "sheet",  # ditto
            }

            results.append(sheet_data)

    return results
Example #16
0
def sheet_spam_dashboard(request):

    from django.contrib.auth.models import User

    if request.method == 'POST':
        return jsonResponse({"error": "Unsupported Method: {}".format(request.method)})

    else:
        date = request.GET.get("date", None)

        if date:
            date = datetime.strptime(date, '%Y-%m-%d')

        else:
            date = request.GET.get("date", datetime.now() - timedelta(days=30))

        earliest_new_user_id = User.objects.filter(date_joined__gte=date).order_by('date_joined')[0].id

        regex = r'.*(?!href=[\'"](\/|http(s)?:\/\/(www\.)?sefaria).+[\'"])(href).*'
        sheets = db.sheets.find({"sources.ref": {"$exists": False}, "dateCreated": {"$gt": date.strftime("%Y-%m-%dT%H:%M:%S.%f")}, "owner": {"$gt": earliest_new_user_id}, "includedRefs": {"$size": 0}, "reviewed": {"$ne": True}, "$or": [{"sources.outsideText": {"$regex": regex}}, {"sources.comment": {"$regex": regex}}, {"sources.outsideBiText.en": {"$regex": regex}}, {"sources.outsideBiText.he": {"$regex": regex}}]})

        sheets_list = []

        for sheet in sheets:
            sheets_list.append({"id": sheet["id"], "title": strip_tags(sheet["title"]), "owner": user_link(sheet["owner"])})

        return render_template(request, 'spam_dashboard.html', None, {
            "title": "Potential Spam Sheets since %s" % date.strftime("%Y-%m-%d"),
            "sheets": sheets_list,
            "type": "sheet",
        })
Example #17
0
def spam_dashboard(request):

    from django.contrib.auth.models import User

    if request.method == 'POST':

        spam_sheet_ids = list(
            map(int, request.POST.getlist("spam_sheets[]", [])))
        reviewed_sheet_ids = list(
            map(int, request.POST.getlist("reviewed_sheets[]", [])))

        db.sheets.update_many({"id": {
            "$in": reviewed_sheet_ids
        }}, {"$set": {
            "reviewed": True
        }})

        spammers = db.sheets.find({
            "id": {
                "$in": spam_sheet_ids
            }
        }, {
            "owner": 1
        }).distinct("owner")

        for spammer in spammers:
            try:
                spammer_account = User.objects.get(id=spammer)
                spammer_account.is_active = False
                spammer_account.save()
            except:
                continue

        db.sheets.delete_many({"id": {"$in": spam_sheet_ids}})

        return render(
            request, 'spam_dashboard.html', {
                "deleted_sheets": len(spam_sheet_ids),
                "sheet_ids": spam_sheet_ids,
                "reviewed_sheets": len(reviewed_sheet_ids),
                "spammers_deactivated": len(spammers)
            })

    else:
        date = request.GET.get("date", None)

        if date:
            date = datetime.strptime(date, '%Y-%m-%d')

        else:
            date = request.GET.get("date", datetime.now() - timedelta(days=30))

        earliest_new_user_id = User.objects.filter(date_joined__gte=date)[0].id

        regex = r'.*(?!href=[\'"](\/|http(s)?:\/\/(www\.)?sefaria).+[\'"])(href).*'
        sheets = db.sheets.find({
            "sources.ref": {
                "$exists": False
            },
            "dateCreated": {
                "$gt": date.strftime("%Y-%m-%dT%H:%M:%S.%f")
            },
            "owner": {
                "$gt": earliest_new_user_id
            },
            "includedRefs": {
                "$size": 0
            },
            "reviewed": {
                "$ne": True
            },
            "$or": [{
                "sources.outsideText": {
                    "$regex": regex
                }
            }, {
                "sources.comment": {
                    "$regex": regex
                }
            }, {
                "sources.outsideBiText.en": {
                    "$regex": regex
                }
            }, {
                "sources.outsideBiText.he": {
                    "$regex": regex
                }
            }]
        })

        sheets_list = []

        for sheet in sheets:
            sheets_list.append({
                "id": sheet["id"],
                "title": strip_tags(sheet["title"]),
                "owner": user_link(sheet["owner"])
            })

        return render(
            request, 'spam_dashboard.html', {
                "title":
                "Potential Spam Sheets since %s" % date.strftime("%Y-%m-%d"),
                "sheets":
                sheets_list,
            })