def text_at_revision(tref, version, lang, revision):
"""
Returns the state of a text (identified by ref/version/lang) at revision number 'revision'
"""
changes = db.history.find({
"ref": tref,
"version": version,
"language": lang
}).sort([['revision', -1]])
current = get_text(tref,
context=0,
commentary=False,
version=version,
lang=lang)
if "error" in current and not current["error"].startswith("No text found"):
return current
textField = "text" if lang == "en" else lang
text = unicode(current.get(textField, ""))
for i in range(changes.count()):
r = changes[i]
if r["revision"] == revision: break
patch = dmp.patch_fromText(r["revert_patch"])
text = dmp.patch_apply(patch, text)[0]
return text
def revert_api(request, ref, lang, version, revision):
"""
API for reverting a text segment to a previous revision.
"""
if not request.user.is_authenticated():
return jsonResponse({"error": "You must be logged in to revert changes."})
if request.method != "POST":
return jsonResponse({"error": "Unsupported HTTP method."})
revision = int(revision)
version = version.replace("_", " ")
ref = norm_ref(ref)
if not ref:
# pass along the error message if norm_ref failed
return jsonResponse(parse_ref(ref))
existing = get_text(ref, commentary=0, version=version, lang=lang)
if "error" in existing:
return jsonResponse(existing)
text = {
"versionTitle": version,
"versionSource": existing["versionSource"] if lang == "en" else existing["heVersionSource"],
"language": lang,
"text": text_at_revision(ref, version, lang, revision)
}
return jsonResponse(save_text(ref, text, request.user.id, type="revert text"))
def record_text_change(tref, version, lang, text, user, **kwargs):
"""
Record a change to a text (ref/version/lang) by user.
"""
# unpack text into smaller segments if necessary (e.g. chapter -> verse)
if isinstance(text, list):
for i in reversed(range(len(text))):
n = i + 1
record_text_change("%s.%d" % (tref, n), version, lang, text[i], user, **kwargs)
return
# get the current state of the text in question
current = get_text(tref, context=0, commentary=False, version=version, lang=lang)
if "error" in current and current["error"].startswith("No text found"):
current = ""
elif "error" in current:
return current
elif lang == "en" and current["text"]:
current = current["text"]
elif lang == "he" and current["he"]:
current = current["he"]
else:
current = ""
# Don't record anything if there's no change.
if not text:
text = ""
if text == current:
return
# create a patch that turns the new version back into the old
backwards_diff = dmp.diff_main(text, current)
patch = dmp.patch_toText(dmp.patch_make(backwards_diff))
# get html displaying edits in this change.
forwards_diff = dmp.diff_main(current, text)
dmp.diff_cleanupSemantic(forwards_diff)
diff_html = dmp.diff_prettyHtml(forwards_diff)
# give this revision a new revision number
revision = next_revision_num()
log = {
"ref": model.Ref(tref).normal(),
"version": version,
"language": lang,
"diff_html": diff_html,
"revert_patch": patch,
"user": user,
"date": datetime.now(),
"revision": revision,
"message": kwargs.get("message", ""),
"rev_type": kwargs.get("type", None) or "edit text" if len(current) else "add text",
"method": kwargs.get("method", "Site")
}
db.history.save(log)
def texts_api(request, ref, lang=None, version=None):
if request.method == "GET":
cb = request.GET.get("callback", None)
context = int(request.GET.get("context", 1))
commentary = bool(int(request.GET.get("commentary", True)))
version = version.replace("_", " ") if version else None
text = get_text(ref, version=version, lang=lang, commentary=commentary, context=context)
if "error" in text:
return jsonResponse(text, cb)
if "commentary" in text:
# If this is a spanning ref it can't handle commmentary,
# so check if the field is actually present
notes = get_notes(ref, uid=request.user.id, context=1)
text["commentary"] += notes
return jsonResponse(text, cb)
if request.method == "POST":
j = request.POST.get("json")
if not j:
return jsonResponse({"error": "Missing 'json' parameter in post data."})
# Parameters to suppress some costly operations after save
count_after = int(request.GET.get("count_after", 1))
index_after = int(request.GET.get("index_after", 1))
if not request.user.is_authenticated():
key = request.POST.get("apikey")
if not key:
return jsonResponse({"error": "You must be logged in or use an API key to save texts."})
apikey = db.apikeys.find_one({"key": key})
if not apikey:
return jsonResponse({"error": "Unrecognized API key."})
response = save_text(ref, json.loads(j), apikey["uid"], method="API", count_after=count_after, index_after=index_after)
return jsonResponse(response)
else:
@csrf_protect
def protected_post(request):
response = save_text(ref, json.loads(j), request.user.id, count_after=count_after, index_after=index_after)
return jsonResponse(response)
return protected_post(request)
return jsonResponse({"error": "Unsuported HTTP method."})
def format_link_object_for_client(link, with_text, ref, pos=None):
"""
:param link: Link object
:param ref: Ref object of the source of the link
:param pos: Optional position of the Ref in the Link. If not passed, it will be derived from the first two arguments.
:return: Dict
"""
com = {}
# The text we're asked to get links to
anchorRef = model.Ref(link.refs[pos])
# The link we found to anchorRef
linkRef = model.Ref(link.refs[(pos + 1) % 2])
com["_id"] = str(link._id)
com["category"] = linkRef.type
com["type"] = link.type
com["ref"] = linkRef.tref
com["anchorRef"] = anchorRef.normal()
com["sourceRef"] = linkRef.normal()
com["anchorVerse"] = anchorRef.sections[-1]
com["commentaryNum"] = linkRef.sections[-1] if linkRef.type == "Commentary" else 0
com["anchorText"] = getattr(link, "anchorText", "")
if with_text:
from sefaria.texts import get_text
text = get_text(linkRef.normal(), context=0, commentary=False)
com["text"] = text["text"] if text["text"] else ""
com["he"] = text["he"] if text["he"] else ""
# strip redundant verse ref for commentators
# if the ref we're looking for appears exactly in the commentary ref, strip redundant info
#todo: this comparison - ref in linkRef.normal() - seems brittle. Make it rigorous.
if com["category"] == "Commentary" and ref in linkRef.normal():
com["commentator"] = linkRef.index.commentator
com["heCommentator"] = linkRef.index.heCommentator if getattr(linkRef.index, "heCommentator", None) else com["commentator"]
else:
com["commentator"] = linkRef.book
com["heCommentator"] = linkRef.index.heTitle if getattr(linkRef.index, "heTitle", None) else com["commentator"]
if getattr(linkRef.index, "heTitle", None):
com["heTitle"] = linkRef.index.heTitle
return com
def text_at_revision(tref, version, lang, revision):
"""
Returns the state of a text (identified by ref/version/lang) at revision number 'revision'
"""
changes = db.history.find({"ref": tref, "version": version, "language": lang}).sort([['revision', -1]])
current = get_text(tref, context=0, commentary=False, version=version, lang=lang)
if "error" in current and not current["error"].startswith("No text found"):
return current
textField = "text" if lang == "en" else lang
text = unicode(current.get(textField, ""))
for i in range(changes.count()):
r = changes[i]
if r["revision"] == revision: break
patch = dmp.patch_fromText(r["revert_patch"])
text = dmp.patch_apply(patch, text)[0]
return text
def get_links(tref, with_text=True):
"""
Return a list of links tied to 'ref' in client format.
If with_text, retrieve texts for each link.
"""
links = []
oref = model.Ref(tref)
nRef = oref.normal()
reRef = oref.regex()
# for storing all the section level texts that need to be looked up
texts = {}
linkset = model.LinkSet({"refs": {"$regex": reRef}})
# For all links that mention ref (in any position)
for link in linkset:
# each link contins 2 refs in a list
# find the position (0 or 1) of "anchor", the one we're getting links for
pos = 0 if re.match(reRef, link.refs[0]) else 1
try:
com = format_link_object_for_client(link, False, nRef, pos)
except InputError:
# logger.warning("Bad link: {} - {}".format(link.refs[0], link.refs[1]))
continue
# Rather than getting text with each link, walk through all links here,
# caching text so that redudant DB calls can be minimized
if with_text:
com_oref = model.Ref(com["ref"])
top_nref = com_oref.top_section_ref().normal()
# Lookup and save top level text, only if we haven't already
if top_nref not in texts:
texts[top_nref] = get_text(top_nref, context=0, commentary=False, pad=False)
sections, toSections = com_oref.sections[1:], com_oref.toSections[1:]
com["text"] = grab_section_from_text(sections, texts[top_nref]["text"], toSections)
com["he"] = grab_section_from_text(sections, texts[top_nref]["he"], toSections)
links.append(com)
return links
def edit_text(request, ref=None, lang=None, version=None, new_name=None):
"""
Opens a view directly to adding, editing or translating a given text.
"""
if ref is not None:
version = version.replace("_", " ") if version else None
text = get_text(ref, lang=lang, version=version)
text["mode"] = request.path.split("/")[1]
initJSON = json.dumps(text)
else:
new_name = new_name.replace("_", " ") if new_name else new_name
initJSON = json.dumps({"mode": "add new", "title": new_name})
titles = json.dumps(get_text_titles())
page_title = "%s %s" % (text["mode"].capitalize(), ref) if ref else "Add a New Text"
email = request.user.email if request.user.is_authenticated() else ""
return render_to_response('reader.html',
{'titles': titles,
'initJSON': initJSON,
'page_title': page_title,
'email': email},
RequestContext(request))
def translation_flow(request, ref):
"""
Assign a user a paritcular bit of text to translate within 'ref',
either a text title or category.
"""
ref = ref.replace("_", " ")
generic_response = { "title": "Help Translate %s" % ref, "content": "" }
categories = get_text_categories()
next_text = None
next_section = None
# expire old locks before checking for a currently unlocked text
sefaria.locks.expire_locks()
pRef = parse_ref(ref, pad=False)
if "error" not in pRef and len(pRef["sections"]) == 0:
# ref is an exact text Title
text = norm_ref(ref)
# normalize URL
if request.path != "/translate/%s" % url_ref(text):
return redirect("/translate/%s" % url_ref(text), permanent=True)
# Check for completion
if get_percent_available(text) == 100:
generic_response["content"] = "<h3>Sefaria now has a complete translation of %s</h3>But you can still contribute in other ways.</h3> <a href='/contribute'>Learn More.</a>" % ref
return render_to_response('static/generic.html', generic_response, RequestContext(request))
if "random" in request.GET:
# choose a ref from a random section within this text
skip = int(request.GET.get("skip")) if "skip" in request.GET else None
assigned_ref = random_untranslated_ref_in_text(text, skip=skip)
if assigned_ref:
next_section = parse_ref(assigned_ref)["sections"][0]
elif "section" in request.GET:
# choose the next ref within the specified section
next_section = int(request.GET["section"])
assigned_ref = next_untranslated_ref_in_text(text, section=next_section)
else:
# choose the next ref in this text in order
assigned_ref = next_untranslated_ref_in_text(text)
if not assigned_ref:
generic_response["content"] = "All remaining sections in %s are being worked on by other contributors. Work on <a href='/translate/%s'>another text</a> for now." % (text, ref)
return render_to_response('static/generic.html', generic_response, RequestContext(request))
elif "error" not in pRef and len(pRef["sections"]) > 0:
# ref is a citation to a particular location in a text
# for now, send this to the edit_text view
return edit_text(request, ref)
elif "error" in pRef and ref in categories:
# ref is a text Category
cat = ref
# Check for completion
if get_percent_available(cat) == 100:
generic_response["content"] = "<h3>Sefaria now has a complete translation of %s</h3>But you can still contribute in other ways.</h3> <a href='/contribute'>Learn More.</a>" % ref
return render_to_response('static/generic.html', generic_response, RequestContext(request))
if "random" in request.GET:
# choose a random text from this cateogory
skip = int(request.GET.get("skip")) if "skip" in request.GET else None
text = random_untranslated_text_in_category(cat, skip=skip)
assigned_ref = next_untranslated_ref_in_text(text)
next_text = text
elif "text" in request.GET:
# choose the next text requested in URL
text = norm_ref(request.GET["text"])
next_text = text
if get_percent_available(text) == 100:
generic_response["content"] = "%s is complete! Work on <a href='/translate/%s'>another text</a>." % (next, ref)
return render_to_response('static/generic.html', generic_response, RequestContext(request))
assigned_ref = next_untranslated_ref_in_text(text)
if "error" in assigned_ref:
generic_response["content"] = "All remaining sections in %s are being worked on by other contributors. Work on <a href='/translate/%s'>another text</a> for now." % (next, ref)
return render_to_response('static/generic.html', generic_response, RequestContext(request))
else:
# choose the next text in order
skip = 0
assigned_ref = {"error": "haven't chosen yet"}
# TODO -- need an escape valve here
while "error" in assigned_ref:
text = next_untranslated_text_in_category(cat, skip=skip)
assigned_ref = next_untranslated_ref_in_text(text)
skip += 1
else:
# we don't know what this is
generic_response["content"] = "<b>%s</b> isn't a known text or category.<br>But you can still contribute in other ways.</h3> <a href='/contribute'>Learn More.</a>" % (ref)
return render_to_response('static/generic.html', generic_response, RequestContext(request))
# get the assigned text
assigned = get_text(assigned_ref, context=0, commentary=False)
# Put a lock on this assignment
user = request.user.id if request.user.is_authenticated() else 0
sefaria.locks.set_lock(assigned_ref, "en", "Sefaria Community Translation", user)
# if the assigned text is actually empty, run this request again
# but leave the new lock in place to skip over it
if "he" not in assigned or not len(assigned["he"]):
return translation_flow(request, ref)
# get percentage and remaining counts
# percent = get_percent_available(assigned["book"])
translated = get_translated_count_by_unit(assigned["book"], unit=assigned["sectionNames"][-1])
remaining = get_untranslated_count_by_unit(assigned["book"], unit=assigned["sectionNames"][-1])
percent = 100 * translated / float(translated + remaining)
return render_to_response('translate_campaign.html',
{"title": "Help Translate %s" % ref,
"base_ref": ref,
"assigned_ref": assigned_ref,
"assigned_ref_url": url_ref(assigned_ref),
"assigned_text": assigned["he"],
"assigned_segment_name": assigned["sectionNames"][-1],
"assigned": assigned,
"translated": translated,
"remaining": remaining,
"percent": percent,
"thanks": "thank" in request.GET,
"random_param": "&skip=%d" % assigned["sections"][0] if request.GET.get("random") else "",
"next_text": next_text,
"next_section": next_section,
},
RequestContext(request))
def reader(request, ref, lang=None, version=None):
# Redirect to standard URLs
# Let unknown refs pass through
uref = url_ref(ref)
if uref and ref != uref:
url = "/" + uref
if lang and version:
url += "/%s/%s" % (lang, version)
response = redirect(url, permanent=True)
params = request.GET.urlencode()
response['Location'] += "?%s" % params if params else ""
return response
# BANDAID - return the first section only of a spanning ref
pRef = parse_ref(ref)
if "error" not in pRef and is_spanning_ref(pRef):
ref = split_spanning_ref(pRef)[0]
url = "/" + ref
if lang and version:
url += "/%s/%s" % (lang, version)
response = redirect(url)
params = request.GET.urlencode()
response['Location'] += "?%s" % params if params else ""
return response
version = version.replace("_", " ") if version else None
text = get_text(ref, lang=lang, version=version)
if not "error" in text:
notes = get_notes(ref, uid=request.user.id, context=1)
text["commentary"] += notes
initJSON = json.dumps(text)
lines = True if "error" in text or text["type"] not in ('Tanach', 'Talmud') or text["book"] == "Psalms" else False
email = request.user.email if request.user.is_authenticated() else ""
zippedText = map(None, text["text"], text["he"]) if not "error" in text else []
# Pull language setting from cookie or Accept-Lanugage header
langMode = request.COOKIES.get('langMode') or request.LANGUAGE_CODE or 'en'
langMode = 'he' if langMode == 'he-il' else langMode
# URL parameter trumps cookie
langMode = request.GET.get("lang", langMode)
langMode = "bi" if langMode in ("he-en", "en-he") else langMode
# Don't allow languages other than what we currently handle
langMode = 'en' if langMode not in ('en', 'he', 'bi') else langMode
# Substitue language mode if text not available in that language
if not "error" in text:
if is_text_empty(text["text"]) and not langMode == "he":
langMode = "he"
if is_text_empty(text["he"]) and not langMode == "en":
langMode = "en"
langClass = {"en": "english", "he": "hebrew", "bi": "bilingual heLeft"}[langMode]
return render_to_response('reader.html',
{'text': text,
'initJSON': initJSON,
'zippedText': zippedText,
'lines': lines,
'langClass': langClass,
'page_title': norm_ref(ref) or "Unknown Text",
'title_variants': "(%s)" % ", ".join(text.get("titleVariants", []) + [text.get("heTitle", "")]),
'email': email},
RequestContext(request))
def parashat_hashavua_api(request):
callback = request.GET.get("callback", None)
p = sefaria.calendars.this_weeks_parasha(datetime.now())
p["date"] = p["date"].isoformat()
p.update(get_text(p["ref"]))
return jsonResponse(p, callback)
row[2].strip(),
"language":
version_langs[row[3]],
"version":
row[3],
}
except Exception, e:
print "ERROR Importing: %s" % e
continue
valid = validate_review(review)
if "error" in valid:
print "ERROR Validating: %s" % valid["error"]
continue
text = get_text(review["ref"],
context=1,
commentary=False,
version=review["version"],
lang=review["language"])
field = "text" if review["language"] == "en" else "he"
if not text[field]:
print "ERROR Matching: No text found for %s, %s" % (
review["ref"], review["version"])
versions = get_version_list(review["ref"])
print "Versions: %s" % ", ".join([
v["versionTitle"]
for v in versions if v["language"] == review["language"]
])
db.history.save(review)
header = reviews.next()
for row in reviews:
try:
review = {
"user": reviewers[row[1]],
"date": datetime.strptime(row[0], "%m/%d/%Y").replace(hour=18, minute=59),
"rev_type": "review",
"score": float(row[4]),
"comment": row[5],
"ref": row[2].strip(),
"language": version_langs[row[3]],
"version": row[3],
}
except Exception, e:
print "ERROR Importing: %s" % e
continue
valid = validate_review(review)
if "error" in valid:
print "ERROR Validating: %s" % valid["error"]
continue
text = get_text(review["ref"], context=1, commentary=False, version=review["version"], lang=review["language"])
field = "text" if review["language"] == "en" else "he"
if not text[field]:
print "ERROR Matching: No text found for %s, %s" % (review["ref"], review["version"])
versions = get_version_list(review["ref"])
print "Versions: %s" % ", ".join([v["versionTitle"] for v in versions if v["language"] == review["language"]])
db.history.save(review)
import os
p = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.insert(0, p)
sys.path.insert(0, p + "/sefaria")
import pymongo
from sefaria.texts import get_text, generate_refs_list
from sefaria.settings import *
connection = pymongo.Connection()
db = connection[SEFARIA_DB]
if SEFARIA_DB_USER and SEFARIA_DB_PASSWORD:
db.authenticate(SEFARIA_DB_USER, SEFARIA_DB_PASSWORD)
chapters = generate_refs_list({"title": {"$regex": "Onkelos"}})
for ref in chapters:
text = get_text(ref, commentary=0)
verses = len(text["he"])
for i in range(verses):
ref1 = "%s:%d" % (ref, i+1)
ref2 = ref1.replace("Onkelos ", "")
link = {
"refs": [ref1, ref2],
"type": "targum"
}
db.links.save(link)
# -*- coding: utf-8 -*-
import sys
import os
p = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, p)
sys.path.insert(0, p + "/sefaria")
import pymongo
from sefaria.texts import get_text, generate_refs_list
from sefaria.settings import *
connection = pymongo.Connection()
db = connection[SEFARIA_DB]
if SEFARIA_DB_USER and SEFARIA_DB_PASSWORD:
db.authenticate(SEFARIA_DB_USER, SEFARIA_DB_PASSWORD)
chapters = generate_refs_list({"title": {"$regex": "Targum Jonathan"}})
for ref in chapters:
text = get_text(ref, commentary=0)
verses = len(text["he"])
for i in range(verses):
ref1 = "%s:%d" % (ref, i + 1)
ref2 = ref1.replace("Targum Jonathan on ", "")
link = {"refs": [ref1, ref2], "type": "targum"}
db.links.save(link)
def record_text_change(tref, version, lang, text, user, **kwargs):
"""
Record a change to a text (ref/version/lang) by user.
"""
# unpack text into smaller segments if necessary (e.g. chapter -> verse)
if isinstance(text, list):
for i in reversed(range(len(text))):
n = i + 1
record_text_change("%s.%d" % (tref, n), version, lang, text[i],
user, **kwargs)
return
# get the current state of the text in question
current = get_text(tref,
context=0,
commentary=False,
version=version,
lang=lang)
if "error" in current and current["error"].startswith("No text found"):
current = ""
elif "error" in current:
return current
elif lang == "en" and current["text"]:
current = current["text"]
elif lang == "he" and current["he"]:
current = current["he"]
else:
current = ""
# Don't record anything if there's no change.
if not text:
text = ""
if text == current:
return
# create a patch that turns the new version back into the old
backwards_diff = dmp.diff_main(text, current)
patch = dmp.patch_toText(dmp.patch_make(backwards_diff))
# get html displaying edits in this change.
forwards_diff = dmp.diff_main(current, text)
dmp.diff_cleanupSemantic(forwards_diff)
diff_html = dmp.diff_prettyHtml(forwards_diff)
# give this revision a new revision number
revision = next_revision_num()
log = {
"ref": model.Ref(tref).normal(),
"version": version,
"language": lang,
"diff_html": diff_html,
"revert_patch": patch,
"user": user,
"date": datetime.now(),
"revision": revision,
"message": kwargs.get("message", ""),
"rev_type": kwargs.get("type", None) or "edit text"
if len(current) else "add text",
"method": kwargs.get("method", "Site")
}
db.history.save(log)