def text_at_revision(tref, version, lang, revision):
    """
    Returns the state of a text (identified by ref/version/lang) at revision number 'revision'
    """
    changes = db.history.find({
        "ref": tref,
        "version": version,
        "language": lang
    }).sort([['revision', -1]])
    current = get_text(tref,
                       context=0,
                       commentary=False,
                       version=version,
                       lang=lang)
    if "error" in current and not current["error"].startswith("No text found"):
        return current

    textField = "text" if lang == "en" else lang
    text = unicode(current.get(textField, ""))

    for i in range(changes.count()):
        r = changes[i]
        if r["revision"] == revision: break
        patch = dmp.patch_fromText(r["revert_patch"])
        text = dmp.patch_apply(patch, text)[0]

    return text
Exemple #2
0
def revert_api(request, ref, lang, version, revision):
	"""
	API for reverting a text segment to a previous revision.
	"""
	if not request.user.is_authenticated():
		return jsonResponse({"error": "You must be logged in to revert changes."})

	if request.method != "POST":
		return jsonResponse({"error": "Unsupported HTTP method."})

	revision = int(revision)
	version = version.replace("_", " ")
	ref = norm_ref(ref)
	if not ref:
		# pass along the error message if norm_ref failed
		return jsonResponse(parse_ref(ref))

	existing = get_text(ref, commentary=0, version=version, lang=lang)
	if "error" in existing:
		return jsonResponse(existing)

	text = {
		"versionTitle": version,
		"versionSource": existing["versionSource"] if lang == "en" else existing["heVersionSource"],
		"language": lang,
		"text": text_at_revision(ref, version, lang, revision)
	}

	return jsonResponse(save_text(ref, text, request.user.id, type="revert text"))
def record_text_change(tref, version, lang, text, user, **kwargs):
    """
    Record a change to a text (ref/version/lang) by user.
    """

    # unpack text into smaller segments if necessary (e.g. chapter -> verse)
    if isinstance(text, list):
        for i in reversed(range(len(text))):
            n = i + 1
            record_text_change("%s.%d" % (tref, n), version, lang, text[i], user, **kwargs)
        return

    # get the current state of the text in question
    current = get_text(tref, context=0, commentary=False, version=version, lang=lang)
    if "error" in current and current["error"].startswith("No text found"):
        current = ""
    elif "error" in current:
        return current
    elif lang == "en" and current["text"]:
        current = current["text"]
    elif lang == "he" and current["he"]:
        current = current["he"]
    else:
        current = ""

    # Don't record anything if there's no change.
    if not text:
        text = ""
    if text == current:
        return

    # create a patch that turns the new version back into the old
    backwards_diff = dmp.diff_main(text, current)
    patch = dmp.patch_toText(dmp.patch_make(backwards_diff))
    # get html displaying edits in this change.
    forwards_diff = dmp.diff_main(current, text)
    dmp.diff_cleanupSemantic(forwards_diff)
    diff_html = dmp.diff_prettyHtml(forwards_diff)

    # give this revision a new revision number
    revision = next_revision_num()

    log = {
        "ref": model.Ref(tref).normal(),
        "version": version,
        "language": lang,
        "diff_html": diff_html,
        "revert_patch": patch,
        "user": user,
        "date": datetime.now(),
        "revision": revision,
        "message": kwargs.get("message", ""),
        "rev_type": kwargs.get("type", None) or "edit text" if len(current) else "add text",
        "method": kwargs.get("method", "Site")
    }

    db.history.save(log)
Exemple #4
0
def texts_api(request, ref, lang=None, version=None):
	if request.method == "GET":
		cb = request.GET.get("callback", None)
		context = int(request.GET.get("context", 1))
		commentary = bool(int(request.GET.get("commentary", True)))
		version = version.replace("_", " ") if version else None

		text = get_text(ref, version=version, lang=lang, commentary=commentary, context=context)
		
		if "error" in text:
			return jsonResponse(text, cb)

		if "commentary" in text:
			# If this is a spanning ref it can't handle commmentary,
			# so check if the field is actually present 
			notes = get_notes(ref, uid=request.user.id, context=1)
			text["commentary"] += notes

		return jsonResponse(text, cb)


	if request.method == "POST":
		j = request.POST.get("json")
		if not j:
			return jsonResponse({"error": "Missing 'json' parameter in post data."})
		
		# Parameters to suppress some costly operations after save
		count_after = int(request.GET.get("count_after", 1))
		index_after = int(request.GET.get("index_after", 1))
		if not request.user.is_authenticated():
			key = request.POST.get("apikey")
			if not key:
				return jsonResponse({"error": "You must be logged in or use an API key to save texts."})
			apikey = db.apikeys.find_one({"key": key})
			if not apikey:
				return jsonResponse({"error": "Unrecognized API key."})
			response = save_text(ref, json.loads(j), apikey["uid"], method="API", count_after=count_after, index_after=index_after)
			return jsonResponse(response)
		else:
			@csrf_protect
			def protected_post(request):
				response = save_text(ref, json.loads(j), request.user.id, count_after=count_after, index_after=index_after)
				return jsonResponse(response)
			return protected_post(request)

	return jsonResponse({"error": "Unsuported HTTP method."})
def format_link_object_for_client(link, with_text, ref, pos=None):
    """
    :param link: Link object
    :param ref: Ref object of the source of the link
    :param pos: Optional position of the Ref in the Link.  If not passed, it will be derived from the first two arguments.
    :return: Dict
    """
    com = {}

    # The text we're asked to get links to
    anchorRef = model.Ref(link.refs[pos])

    # The link we found to anchorRef
    linkRef = model.Ref(link.refs[(pos + 1) % 2])

    com["_id"]           = str(link._id)
    com["category"]      = linkRef.type
    com["type"]          = link.type
    com["ref"]           = linkRef.tref
    com["anchorRef"]     = anchorRef.normal()
    com["sourceRef"]     = linkRef.normal()
    com["anchorVerse"]   = anchorRef.sections[-1]
    com["commentaryNum"] = linkRef.sections[-1] if linkRef.type == "Commentary" else 0
    com["anchorText"]    = getattr(link, "anchorText", "")

    if with_text:
        from sefaria.texts import get_text
        text             = get_text(linkRef.normal(), context=0, commentary=False)
        com["text"]      = text["text"] if text["text"] else ""
        com["he"]        = text["he"] if text["he"] else ""

    # strip redundant verse ref for commentators
    # if the ref we're looking for appears exactly in the commentary ref, strip redundant info
    #todo: this comparison - ref in linkRef.normal() - seems brittle.  Make it rigorous.
    if com["category"] == "Commentary" and ref in linkRef.normal():
        com["commentator"] = linkRef.index.commentator
        com["heCommentator"] = linkRef.index.heCommentator if getattr(linkRef.index, "heCommentator", None) else com["commentator"]
    else:
        com["commentator"] = linkRef.book
        com["heCommentator"] = linkRef.index.heTitle if getattr(linkRef.index, "heTitle", None) else com["commentator"]

    if getattr(linkRef.index, "heTitle", None):
        com["heTitle"] = linkRef.index.heTitle

    return com
def text_at_revision(tref, version, lang, revision):
    """
    Returns the state of a text (identified by ref/version/lang) at revision number 'revision'
    """
    changes = db.history.find({"ref": tref, "version": version, "language": lang}).sort([['revision', -1]])
    current = get_text(tref, context=0, commentary=False, version=version, lang=lang)
    if "error" in current and not current["error"].startswith("No text found"):
        return current

    textField = "text" if lang == "en" else lang
    text = unicode(current.get(textField, ""))

    for i in range(changes.count()):
        r = changes[i]
        if r["revision"] == revision: break
        patch = dmp.patch_fromText(r["revert_patch"])
        text = dmp.patch_apply(patch, text)[0]

    return text
def get_links(tref, with_text=True):
    """
    Return a list of links tied to 'ref' in client format.
    If with_text, retrieve texts for each link.
    """
    links = []
    oref = model.Ref(tref)
    nRef = oref.normal()
    reRef = oref.regex()

    # for storing all the section level texts that need to be looked up
    texts = {}

    linkset = model.LinkSet({"refs": {"$regex": reRef}})
    # For all links that mention ref (in any position)
    for link in linkset:
        # each link contins 2 refs in a list
        # find the position (0 or 1) of "anchor", the one we're getting links for
        pos = 0 if re.match(reRef, link.refs[0]) else 1
        try:
            com = format_link_object_for_client(link, False, nRef, pos)
        except InputError:
            # logger.warning("Bad link: {} - {}".format(link.refs[0], link.refs[1]))
            continue

        # Rather than getting text with each link, walk through all links here,
        # caching text so that redudant DB calls can be minimized
        if with_text:
            com_oref = model.Ref(com["ref"])
            top_nref = com_oref.top_section_ref().normal()

            # Lookup and save top level text, only if we haven't already
            if top_nref not in texts:
                texts[top_nref] = get_text(top_nref, context=0, commentary=False, pad=False)

            sections, toSections = com_oref.sections[1:], com_oref.toSections[1:]
            com["text"] = grab_section_from_text(sections, texts[top_nref]["text"], toSections)
            com["he"]   = grab_section_from_text(sections, texts[top_nref]["he"],   toSections)

        links.append(com)

    return links
Exemple #8
0
def edit_text(request, ref=None, lang=None, version=None, new_name=None):
	"""
	Opens a view directly to adding, editing or translating a given text.
	"""
	if ref is not None:
		version = version.replace("_", " ") if version else None
		text = get_text(ref, lang=lang, version=version)
		text["mode"] = request.path.split("/")[1] 
		initJSON = json.dumps(text)
	else:
		new_name = new_name.replace("_", " ") if new_name else new_name
		initJSON = json.dumps({"mode": "add new", "title": new_name})

	titles = json.dumps(get_text_titles())
	page_title = "%s %s" % (text["mode"].capitalize(), ref) if ref else "Add a New Text" 
	email = request.user.email if request.user.is_authenticated() else ""


	return render_to_response('reader.html', 
							 {'titles': titles,
							 'initJSON': initJSON, 
							 'page_title': page_title,
							 'email': email}, 
							 RequestContext(request))
Exemple #9
0
def translation_flow(request, ref):
	"""
	Assign a user a paritcular bit of text to translate within 'ref',
	either a text title or category. 
	"""
	ref = ref.replace("_", " ")
	generic_response = { "title": "Help Translate %s" % ref, "content": "" }
	categories = get_text_categories()
	next_text = None
	next_section = None

	# expire old locks before checking for a currently unlocked text
	sefaria.locks.expire_locks()

	pRef = parse_ref(ref, pad=False)
	if "error" not in pRef and len(pRef["sections"]) == 0:
		# ref is an exact text Title
		text = norm_ref(ref)

		# normalize URL
		if request.path != "/translate/%s" % url_ref(text):
			return redirect("/translate/%s" % url_ref(text), permanent=True)

		# Check for completion
		if get_percent_available(text) == 100:
			generic_response["content"] = "<h3>Sefaria now has a complete translation of %s</h3>But you can still contribute in other ways.</h3> <a href='/contribute'>Learn More.</a>" % ref
			return render_to_response('static/generic.html', generic_response, RequestContext(request))

		if "random" in request.GET:
			# choose a ref from a random section within this text
			skip = int(request.GET.get("skip")) if "skip" in request.GET else None
			assigned_ref = random_untranslated_ref_in_text(text, skip=skip)
			
			if assigned_ref:
				next_section = parse_ref(assigned_ref)["sections"][0]
		
		elif "section" in request.GET:
			# choose the next ref within the specified section
			next_section = int(request.GET["section"])
			assigned_ref = next_untranslated_ref_in_text(text, section=next_section)
		
		else:
			# choose the next ref in this text in order
			assigned_ref = next_untranslated_ref_in_text(text)
	
		if not assigned_ref:
			generic_response["content"] = "All remaining sections in %s are being worked on by other contributors. Work on <a href='/translate/%s'>another text</a> for now." % (text, ref)
			return render_to_response('static/generic.html', generic_response, RequestContext(request))
		
	elif "error" not in pRef and len(pRef["sections"]) > 0:
		# ref is a citation to a particular location in a text
		# for now, send this to the edit_text view
		return edit_text(request, ref)
		
	elif "error" in pRef and ref in categories:
		# ref is a text Category
		cat = ref

		# Check for completion
		if get_percent_available(cat) == 100:
			generic_response["content"] = "<h3>Sefaria now has a complete translation of %s</h3>But you can still contribute in other ways.</h3> <a href='/contribute'>Learn More.</a>" % ref
			return render_to_response('static/generic.html', generic_response, RequestContext(request))

		if "random" in request.GET:
			# choose a random text from this cateogory
			skip = int(request.GET.get("skip")) if "skip" in request.GET else None
			text = random_untranslated_text_in_category(cat, skip=skip)
			assigned_ref = next_untranslated_ref_in_text(text)
			next_text = text

		elif "text" in request.GET:
			# choose the next text requested in URL
			text = norm_ref(request.GET["text"])
			next_text = text
			if get_percent_available(text) == 100:
				generic_response["content"] = "%s is complete! Work on <a href='/translate/%s'>another text</a>." % (next, ref)
				return render_to_response('static/generic.html', generic_response, RequestContext(request))
			
			assigned_ref = next_untranslated_ref_in_text(text)
			if "error" in assigned_ref:
				generic_response["content"] = "All remaining sections in %s are being worked on by other contributors. Work on <a href='/translate/%s'>another text</a> for now." % (next, ref)
				return render_to_response('static/generic.html', generic_response, RequestContext(request))

		else:
			# choose the next text in order
			skip = 0
			assigned_ref = {"error": "haven't chosen yet"}
			# TODO -- need an escape valve here
			while "error" in assigned_ref:
				text = next_untranslated_text_in_category(cat, skip=skip)
				assigned_ref = next_untranslated_ref_in_text(text)
				skip += 1
	
	else:
		# we don't know what this is
		generic_response["content"] = "<b>%s</b> isn't a known text or category.<br>But you can still contribute in other ways.</h3> <a href='/contribute'>Learn More.</a>" % (ref)
		return render_to_response('static/generic.html', generic_response, RequestContext(request))


	# get the assigned text
	assigned = get_text(assigned_ref, context=0, commentary=False)

	# Put a lock on this assignment
	user = request.user.id if request.user.is_authenticated() else 0
	sefaria.locks.set_lock(assigned_ref, "en", "Sefaria Community Translation", user)
	
	# if the assigned text is actually empty, run this request again
	# but leave the new lock in place to skip over it
	if "he" not in assigned or not len(assigned["he"]):
		return translation_flow(request, ref)

	# get percentage and remaining counts
	# percent   = get_percent_available(assigned["book"])
	translated = get_translated_count_by_unit(assigned["book"], unit=assigned["sectionNames"][-1])
	remaining = get_untranslated_count_by_unit(assigned["book"], unit=assigned["sectionNames"][-1])
	percent = 100 * translated / float(translated + remaining)


	return render_to_response('translate_campaign.html', 
									{"title": "Help Translate %s" % ref,
									"base_ref": ref,
									"assigned_ref": assigned_ref,
									"assigned_ref_url": url_ref(assigned_ref),
									"assigned_text": assigned["he"],
									"assigned_segment_name": assigned["sectionNames"][-1],
									"assigned": assigned,
									"translated": translated,
									"remaining": remaining,
									"percent": percent,
									"thanks": "thank" in request.GET,
									"random_param": "&skip=%d" % assigned["sections"][0] if request.GET.get("random") else "",
									"next_text": next_text,
									"next_section": next_section,
									},
									RequestContext(request))
Exemple #10
0
def reader(request, ref, lang=None, version=None):

	# Redirect to standard URLs
	# Let unknown refs pass through 
	uref = url_ref(ref)
	if uref and ref != uref:
		url = "/" + uref
		if lang and version:
			url += "/%s/%s" % (lang, version)

		response = redirect(url, permanent=True)
		params = request.GET.urlencode()
		response['Location'] += "?%s" % params if params else ""
		return response

	# BANDAID - return the first section only of a spanning ref
	pRef = parse_ref(ref)
	if "error" not in pRef and is_spanning_ref(pRef):
		ref = split_spanning_ref(pRef)[0]
		url = "/" + ref
		if lang and version:
			url += "/%s/%s" % (lang, version)
		response = redirect(url)
		params = request.GET.urlencode()
		response['Location'] += "?%s" % params if params else ""
		return response

	version = version.replace("_", " ") if version else None
	text = get_text(ref, lang=lang, version=version)
	if not "error" in text:
		notes = get_notes(ref, uid=request.user.id, context=1)
		text["commentary"] += notes
	initJSON = json.dumps(text)
	
	lines = True if "error" in text or text["type"] not in ('Tanach', 'Talmud') or text["book"] == "Psalms" else False
	email = request.user.email if request.user.is_authenticated() else ""
	
	zippedText = map(None, text["text"], text["he"]) if not "error" in text else []

	# Pull language setting from cookie or Accept-Lanugage header
	langMode = request.COOKIES.get('langMode') or request.LANGUAGE_CODE or 'en'
	langMode = 'he' if langMode == 'he-il' else langMode
	# URL parameter trumps cookie
	langMode = request.GET.get("lang", langMode)
	langMode = "bi" if langMode in ("he-en", "en-he") else langMode
	# Don't allow languages other than what we currently handle
	langMode = 'en' if langMode not in ('en', 'he', 'bi') else langMode
	# Substitue language mode if text not available in that language
	if not "error" in text:
		if is_text_empty(text["text"]) and not langMode == "he":
			langMode = "he"
		if is_text_empty(text["he"]) and not langMode == "en":
			langMode = "en"
	langClass = {"en": "english", "he": "hebrew", "bi": "bilingual heLeft"}[langMode]

	return render_to_response('reader.html', 
							 {'text': text,
							 'initJSON': initJSON,
							 'zippedText': zippedText,
							 'lines': lines,
							 'langClass': langClass,
							 'page_title': norm_ref(ref) or "Unknown Text",
							 'title_variants': "(%s)" % ", ".join(text.get("titleVariants", []) + [text.get("heTitle", "")]),
							 'email': email}, 
							 RequestContext(request))
Exemple #11
0
def parashat_hashavua_api(request):
	callback = request.GET.get("callback", None)
	p = sefaria.calendars.this_weeks_parasha(datetime.now())
	p["date"] = p["date"].isoformat()
	p.update(get_text(p["ref"]))
	return jsonResponse(p, callback)
Exemple #12
0
                    row[2].strip(),
                    "language":
                    version_langs[row[3]],
                    "version":
                    row[3],
                }
            except Exception, e:
                print "ERROR Importing: %s" % e
                continue

            valid = validate_review(review)
            if "error" in valid:
                print "ERROR Validating: %s" % valid["error"]
                continue

            text = get_text(review["ref"],
                            context=1,
                            commentary=False,
                            version=review["version"],
                            lang=review["language"])
            field = "text" if review["language"] == "en" else "he"
            if not text[field]:
                print "ERROR Matching: No text found for %s, %s" % (
                    review["ref"], review["version"])
                versions = get_version_list(review["ref"])
                print "Versions: %s" % ", ".join([
                    v["versionTitle"]
                    for v in versions if v["language"] == review["language"]
                ])

            db.history.save(review)
		header = reviews.next()
		for row in reviews:
			
			try:
				review = {
					"user":     reviewers[row[1]],
					"date":     datetime.strptime(row[0], "%m/%d/%Y").replace(hour=18, minute=59),
					"rev_type": "review",
					"score":    float(row[4]),
					"comment":  row[5],
					"ref":      row[2].strip(),
					"language": version_langs[row[3]],
					"version":  row[3],
				}
			except Exception, e:
				print "ERROR Importing: %s" % e
				continue

			valid = validate_review(review)
			if "error" in valid:
				print "ERROR Validating: %s" % valid["error"]
				continue

			text = get_text(review["ref"], context=1, commentary=False, version=review["version"], lang=review["language"])
			field = "text" if review["language"] == "en" else "he"
			if not text[field]:
				print "ERROR Matching: No text found for %s, %s" % (review["ref"], review["version"])
				versions = get_version_list(review["ref"])
				print "Versions: %s" % ", ".join([v["versionTitle"] for v in versions if v["language"] == review["language"]])

			db.history.save(review)
import os
p = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.insert(0, p)
sys.path.insert(0, p + "/sefaria")

import pymongo

from sefaria.texts import get_text, generate_refs_list
from sefaria.settings import *

connection = pymongo.Connection()
db = connection[SEFARIA_DB]
if SEFARIA_DB_USER and SEFARIA_DB_PASSWORD:
	db.authenticate(SEFARIA_DB_USER, SEFARIA_DB_PASSWORD)


chapters = generate_refs_list({"title": {"$regex": "Onkelos"}})

for ref in chapters:
	text = get_text(ref, commentary=0)
	verses = len(text["he"])

	for i in range(verses):
		ref1 = "%s:%d" % (ref, i+1)
		ref2 = ref1.replace("Onkelos ", "")

		link = {
			"refs": [ref1, ref2],
			"type": "targum"
		}
		db.links.save(link)
Exemple #15
0
# -*- coding: utf-8 -*-

import sys
import os
p = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, p)
sys.path.insert(0, p + "/sefaria")

import pymongo

from sefaria.texts import get_text, generate_refs_list
from sefaria.settings import *

connection = pymongo.Connection()
db = connection[SEFARIA_DB]
if SEFARIA_DB_USER and SEFARIA_DB_PASSWORD:
    db.authenticate(SEFARIA_DB_USER, SEFARIA_DB_PASSWORD)

chapters = generate_refs_list({"title": {"$regex": "Targum Jonathan"}})

for ref in chapters:
    text = get_text(ref, commentary=0)
    verses = len(text["he"])

    for i in range(verses):
        ref1 = "%s:%d" % (ref, i + 1)
        ref2 = ref1.replace("Targum Jonathan on ", "")

        link = {"refs": [ref1, ref2], "type": "targum"}
        db.links.save(link)
Exemple #16
0
def record_text_change(tref, version, lang, text, user, **kwargs):
    """
    Record a change to a text (ref/version/lang) by user.
    """

    # unpack text into smaller segments if necessary (e.g. chapter -> verse)
    if isinstance(text, list):
        for i in reversed(range(len(text))):
            n = i + 1
            record_text_change("%s.%d" % (tref, n), version, lang, text[i],
                               user, **kwargs)
        return

    # get the current state of the text in question
    current = get_text(tref,
                       context=0,
                       commentary=False,
                       version=version,
                       lang=lang)
    if "error" in current and current["error"].startswith("No text found"):
        current = ""
    elif "error" in current:
        return current
    elif lang == "en" and current["text"]:
        current = current["text"]
    elif lang == "he" and current["he"]:
        current = current["he"]
    else:
        current = ""

    # Don't record anything if there's no change.
    if not text:
        text = ""
    if text == current:
        return

    # create a patch that turns the new version back into the old
    backwards_diff = dmp.diff_main(text, current)
    patch = dmp.patch_toText(dmp.patch_make(backwards_diff))
    # get html displaying edits in this change.
    forwards_diff = dmp.diff_main(current, text)
    dmp.diff_cleanupSemantic(forwards_diff)
    diff_html = dmp.diff_prettyHtml(forwards_diff)

    # give this revision a new revision number
    revision = next_revision_num()

    log = {
        "ref": model.Ref(tref).normal(),
        "version": version,
        "language": lang,
        "diff_html": diff_html,
        "revert_patch": patch,
        "user": user,
        "date": datetime.now(),
        "revision": revision,
        "message": kwargs.get("message", ""),
        "rev_type": kwargs.get("type", None) or "edit text"
        if len(current) else "add text",
        "method": kwargs.get("method", "Site")
    }

    db.history.save(log)