Example #1
0
def get_or_make_summary_node(summary, nodes):
    """
    Returns the node in 'summary' that is named by the list of categories in 'nodes',
    creates the node if it doesn't exist.
    Used recursively on sub-summaries.
    """
    if len(nodes) == 1:
        # Basecase, only need to search through on level
        for node in summary:
            if node.get("category") == nodes[0]:
                return node["contents"]
        # we didn't find it, so let's add it
        summary.append({
            "category": nodes[0],
            "heCategory": hebrew_term(nodes[0]),
            "contents": []
        })
        return summary[-1]["contents"]

    # Look for the first category, or add it, then recur
    for node in summary:
        if node.get("category") == nodes[0]:
            return get_or_make_summary_node(node["contents"], nodes[1:])
    summary.append({
        "category": nodes[0],
        "heCategory": hebrew_term(nodes[0]),
        "contents": []
    })
    return get_or_make_summary_node(summary[-1]["contents"], nodes[1:])
Example #2
0
def get_or_make_summary_node(summary, nodes, contents_only=True, make_if_not_found=True):
    """
    Returns the node in 'summary' that is named by the list of categories in 'nodes',
    If make_if_not_found is true, creates the node if it doesn't exist.
    Used recursively on sub-summaries.
    """
    if len(nodes) == 1:
    #  Basecase, only need to search through one level
        for node in summary:
            if node.get("category") == nodes[0]:
                return node["contents"] if contents_only else node
        # we didn't find it, so let's add it
        if make_if_not_found:
            summary.append({"category": nodes[0], "heCategory": hebrew_term(nodes[0]), "contents": []})
            return summary[-1]["contents"] if contents_only else summary[-1]
        else:
            return None

    # Look for the first category, or add it, then recur
    for node in summary:
        if node.get("category") == nodes[0]:
            return get_or_make_summary_node(node["contents"], nodes[1:], contents_only=contents_only, make_if_not_found=make_if_not_found)

    if make_if_not_found:
        summary.append({"category": nodes[0], "heCategory": hebrew_term(nodes[0]), "contents": []})
        return get_or_make_summary_node(summary[-1]["contents"], nodes[1:], contents_only=contents_only, make_if_not_found=make_if_not_found)
    else:
        return None
Example #3
0
def get_or_make_summary_node(summary, nodes, contents_only=True, make_if_not_found=True):
    """
    Returns the node in 'summary' that is named by the list of categories in 'nodes',
    If make_if_not_found is true, creates the node if it doesn't exist.
    Used recursively on sub-summaries.
    """
    if len(nodes) == 1:
    # Basecase, only need to search through one level
        for node in summary:
            if node.get("category") == nodes[0]:
                return node["contents"] if contents_only else node
        # we didn't find it, so let's add it
        if make_if_not_found:
            summary.append({"category": nodes[0], "heCategory": hebrew_term(nodes[0]), "contents": []})
            return summary[-1]["contents"] if contents_only else summary[-1]
        else:
            return None

    # Look for the first category, or add it, then recur
    for node in summary:
        if node.get("category") == nodes[0]:
            return get_or_make_summary_node(node["contents"], nodes[1:], contents_only=contents_only, make_if_not_found=make_if_not_found)

    if make_if_not_found:
        summary.append({"category": nodes[0], "heCategory": hebrew_term(nodes[0]), "contents": []})
        return get_or_make_summary_node(summary[-1]["contents"], nodes[1:], contents_only=contents_only, make_if_not_found=make_if_not_found)
    else:
        return None
Example #4
0
def format_link_object_for_client(link, with_text, ref, pos=None):
    """
    :param link: Link object
    :param ref: Ref object of the source of the link
    :param pos: Optional position of the Ref in the Link.  If not passed, it will be derived from the first two arguments.
    :return: Dict
    """
    com = {}

    # The text we're asked to get links to
    anchorRef = Ref(link.refs[pos])

    # The link we found to anchorRef
    linkRef = Ref(link.refs[(pos + 1) % 2])

    com["_id"]           = str(link._id)
    com['index_title']   = linkRef.index.title
    com["category"]      = linkRef.primary_category #usually the index's categories[0] or "Commentary".
    com["type"]          = link.type
    com["ref"]           = linkRef.tref
    com["anchorRef"]     = anchorRef.normal()
    com["sourceRef"]     = linkRef.normal()
    com["sourceHeRef"]   = linkRef.he_normal()
    com["anchorVerse"]   = anchorRef.sections[-1] if len(anchorRef.sections) else 0
    com["anchorText"]    = getattr(link, "anchorText", "")

    # Pad out the sections list, so that comparison between comment numbers are apples-to-apples
    lsections = linkRef.sections[:] + [0] * (linkRef.index_node.depth - len(linkRef.sections))
    # Build a decimal comment number based on the last two digits of the section array
    com["commentaryNum"] = lsections[-1] if len(lsections) == 1 \
            else float('{0}.{1:04d}'.format(*lsections[-2:])) if len(lsections) > 1 else 0

    if with_text:
        text             = TextFamily(linkRef, context=0, commentary=False)
        com["text"]      = text.text if isinstance(text.text, basestring) else JaggedTextArray(text.text).flatten_to_array()
        com["he"]        = text.he if isinstance(text.he, basestring) else JaggedTextArray(text.he).flatten_to_array()

    # if the the link is commentary, strip redundant info (e.g. "Rashi on Genesis 4:2" -> "Rashi")
    # this is now simpler, and there is explicit data on the index record for it.
    if com["type"] == "commentary":
        com["linkGroupTitle"] = {
            'en': getattr(linkRef.index, 'collective_title', linkRef.index.title),
            'he': hebrew_term(getattr(linkRef.index, 'collective_title', linkRef.index.get_title("he")))
        }
        com["commentator"] = getattr(linkRef.index, 'collective_title', linkRef.index.title) # TODO: deprecate
        com["heCommentator"] = hebrew_term(getattr(linkRef.index, 'collective_title', linkRef.index.get_title("he"))) # TODO: deprecate
    else:
        com["linkGroupTitle"] = {'en': linkRef.index.title, 'he': linkRef.index.get_title("he")}
        com["commentator"] = linkRef.index.title # TODO: deprecate
        com["heCommentator"] = linkRef.index.get_title("he") # TODO: deprecate

    if com["type"] != "commentary" and com["category"] == "Commentary":
            com["category"] = "Quoting Commentary"
            #add a fix here for quoting commentary appearing together with commentary in s2 panels

    if linkRef.index_node.primary_title("he"):
        com["heTitle"] = linkRef.index_node.primary_title("he")

    return com
Example #5
0
def upload_index(full_text, upload=False):
    """
    :param full_text: Data structure from parse_text()
    :param upload: set to True, otherwise function will do nothing
    """

    if not upload:
        return

    books = [u'Genesis', u'Exodus', u'Leviticus', u'Numbers', u'Deuteronomy']

    # create index record
    record = SchemaNode()
    record.add_title('Chizkuni', 'en', primary=True,)
    record.add_title(u'חזקוני', 'he', primary=True)
    record.key = 'Chizkuni'

    # add nodes
    for book in books:
        node = JaggedArrayNode()
        node.add_title(book, 'en', primary=True)
        node.add_title(hebrew_term(book), 'he', primary=True)
        node.key = book
        node.depth = 3
        node.addressTypes = ['Integer', 'Integer', 'Integer']
        node.sectionNames = ['Chapter', 'Verse', 'Comment']
        record.append(node)
    record.validate()

    index = {
        "title": "Chizkuni",
        "categories": ["Commentary2", "Tanach", "Chizkuni"],
        "schema": record.serialize()
    }
    post_index(index)
Example #6
0
def build_index():

    books = library.get_indexes_in_category('Torah')

    # create index record
    record = SchemaNode()
    record.add_title('Baal HaTurim', 'en', primary=True, )
    record.add_title(u'בעל הטורים', 'he', primary=True, )
    record.key = 'Baal HaTurim'

    # add nodes
    for book in books:
        node = JaggedArrayNode()
        node.add_title(book, 'en', primary=True)
        node.add_title(hebrew_term(book), 'he', primary=True)
        node.key = book
        node.depth = 3
        node.addressTypes = ['Integer', 'Integer', 'Integer']
        node.sectionNames = ['Chapter', 'Verse', 'Comment']
        node.toc_zoom = 2
        record.append(node)
    record.validate()

    index = {
        "title": "Baal HaTurim",
        "categories": ["Commentary2", "Torah", "Baal HaTurim"],
        "schema": record.serialize()
    }
    return index
Example #7
0
def build_index(section_names):
    record = SchemaNode()
    record.add_title(u'Noda BeYehuda', 'en', True)
    record.add_title(u'נודע ביהודה', 'he', True)
    record.key = u'Noda BeYehuda'

    for section in section_names:
        node = JaggedArrayNode()
        node.add_title(section, 'en', True)
        node.add_title(hebrew_term(section), 'he', True)
        node.key = section
        node.depth = 2
        node.addressTypes = ['Integer', 'Integer']
        node.sectionNames = ['Teshuva', 'Paragraph']
        record.append(node)

    record.validate()

    index = {
        'title': 'Noda BeYehuda',
        'categories': ['Responsa'],
        'schema': record.serialize()
    }

    return index
def build_index():

    books = library.get_indexes_in_category('Torah')

    # create index record
    record = SchemaNode()
    record.add_title('Siftei Hakhamim', 'en', primary=True, )
    record.add_title(u'שפתי חכמים', 'he', primary=True, )
    record.key = 'Siftei Hakhamim'

    # add nodes
    for book in books:
        node = JaggedArrayNode()
        node.add_title(book, 'en', primary=True)
        node.add_title(hebrew_term(book), 'he', primary=True)
        node.key = book
        node.depth = 3
        node.addressTypes = ['Integer', 'Integer', 'Integer']
        node.sectionNames = ['Chapter', 'Verse', 'Comment']
        node.toc_zoom = 2
        record.append(node)
    record.validate()

    index = {
        "title": "Siftei Hakhamim",
        "categories": ["Commentary2", "Torah", "Rashi"],
        "schema": record.serialize()
    }
    return index
Example #9
0
def upload_footnote_index():
    """
    Footnotes are uploaded as a commentary2 - Schema with each book a depth 2 jaggedArray
    """

    books = library.get_indexes_in_category('Tanach')

    # create index record
    record = SchemaNode()
    record.add_title('JPS 1985 Footnotes', 'en', primary=True, )
    record.add_title(u'הערות שוליים תרגום 1985 של JPS', 'he', primary=True, )
    record.key = 'JPS 1985 Footnotes'

    # add nodes
    for book in books:
        node = JaggedArrayNode()
        node.add_title(book, 'en', primary=True)
        node.add_title(hebrew_term(book), 'he', primary=True)
        node.key = book
        node.depth = 2
        node.addressTypes = ['Integer', 'Integer']
        node.sectionNames = ['Chapter', 'Footnote']
        record.append(node)
    record.validate()

    index = {
        "title": "JPS 1985 Footnotes",
        "categories": ["Commentary2", "Tanach", "JPS"],
        "schema": record.serialize()
    }
    functions.post_index(index)
Example #10
0
    def create_daf_connection_story(cls, **kwargs):
        # todo: use reccomendation engine
        daf_ref = daf_yomi_ref()
        connection_link, connection_ref = random_connection_to(daf_ref)

        if not connection_ref:
            return

        category = connection_ref.index.categories[0]

        mustHave = []
        if not connection_ref.is_text_translated():
            mustHave += ["readsHebrew"]

        if category == "Talmud":
            title = {"en": "Related Passage", "he": "סוגיה קשורה"}
        else:
            title = {"en": category + " on the Daf", "he": hebrew_term(category) + " " + "על הדף"}

        try:
            cls.generate_story(
                refs = [connection_link.ref_opposite(connection_ref).normal(), connection_ref.normal()],
                title=title,
                lead={'en': 'Daf Yomi', 'he': "דף יומי"},
                mustHave=mustHave,
                **kwargs
            ).save()
        except AttributeError:
            # connection_link.ref_opposite(connection_ref).normal() err's out ... why?
            return
Example #11
0
        def _create_parasha_verse_commentator_story(parasha_obj, mustHave=None, **kwargs):
            from sefaria.utils.calendars import make_parashah_response_from_calendar_entry
            from . import ref_data

            mustHave = mustHave or []

            cal = make_parashah_response_from_calendar_entry(parasha_obj)[0]
            parasha_ref = text.Ref(parasha_obj["ref"])

            top_ref = ref_data.RefDataSet.from_ref(parasha_ref).nth_ref(iteration)

            commentary_ref = random_commentary_on(top_ref)
            if not commentary_ref:
                return

            if not commentary_ref.is_text_translated():
                mustHave += ["readsHebrew"]
            commentator = commentary_ref.index.collective_title

            cls.generate_story(
                refs = [top_ref.normal(), commentary_ref.normal()],
                title={"en": commentator + " on " + cal["displayValue"]["en"], "he": hebrew_term(commentator) + " על " + cal["displayValue"]["he"]},
                lead={"en": "Weekly Torah Portion", "he": 'פרשת השבוע'},
                mustHave=mustHave,
                **kwargs
            ).save()
Example #12
0
 def _data_object(cls, **kwargs):
     normal_topics = [text.Term.normalize(topics) for topics in kwargs.get("topics")]
     # todo: handle possibility of Hebrew terms trending.
     return {
         "topics": [{"en": topic, "he": hebrew_term(topic)} for topic in normal_topics],
         "title": kwargs.get("title", {"en": "Trending Recently", "he": u"פופולרי"}),
         "lead": kwargs.get("lead", {"en": "Topics", "he": u"נושאים"})
     }
Example #13
0
 def generate_topic_story(cls, topic, **kwargs):
     t = text.Term.normalize(topic)
     return cls.generate_story(sheet_ids=cls._get_topic_sheet_ids(topic),
                               title={
                                   "en": t,
                                   "he": hebrew_term(t)
                               },
                               **kwargs)
Example #14
0
def get_or_make_summary_node(summary, nodes):
    """
    Returns the node in 'summary' that is named by the list of categories in 'nodes',
    creates the node if it doesn't exist.
    Used recursively on sub-summaries.
    """
    if len(nodes) == 1:
    # Basecase, only need to search through on level
        for node in summary:
            if node.get("category") == nodes[0]:
                return node["contents"]
        # we didn't find it, so let's add it
        summary.append({"category": nodes[0], "heCategory": hebrew_term(nodes[0]), "contents": []})
        return summary[-1]["contents"]

    # Look for the first category, or add it, then recur
    for node in summary:
        if node.get("category") == nodes[0]:
            return get_or_make_summary_node(node["contents"], nodes[1:])
    summary.append({"category": nodes[0], "heCategory": hebrew_term(nodes[0]), "contents": []})
    return get_or_make_summary_node(summary[-1]["contents"], nodes[1:])
Example #15
0
def build_index(parser):

    assert isinstance(parser, PeneiDavid)

    root = SchemaNode()
    root.add_title('Penei David', 'en', primary=True)
    root.add_title(u'פני דוד', 'he', primary=True)
    root.key = 'Penei David'

    title_node = JaggedArrayNode()
    title_node.add_title('Title Page', 'en', primary=True)
    title_node.add_title(u'עמוד שער', 'he', primary=True)
    title_node.key = 'Title Page'
    title_node.depth = 1
    title_node.addressTypes = ['Integer']
    title_node.sectionNames = ["Paragraph"]
    root.append(title_node)

    # add book nodes
    for book in parser.book_names:
        book_node = SchemaNode()
        book_node.add_title(book, 'en', primary=True)
        book_node.add_title(hebrew_term(book), 'he', primary=True)
        book_node.key = book

        # add parsha nodes
        for parsha in parser.parsha_by_book[book]:
            parsha_node = JaggedArrayNode()
            parsha_node.add_title(parsha, 'en', primary=True)
            parsha_node.add_title(parser.parsha_names_translated[parsha],
                                  'he',
                                  primary=True)
            parsha_node.key = parsha
            parsha_node.depth = 2
            parsha_node.addressTypes = ['Integer', 'Integer']
            parsha_node.sectionNames = ['Comment', 'Paragraph']
            book_node.append(parsha_node)

        root.append(book_node)
    root.validate()

    index = {
        "title": "Penei David",
        "categories": ["Commentary2", "Torah", "Penei David"],
        "schema": root.serialize()
    }
    return index
Example #16
0
    def _data_object(cls, **kwargs):
        t = kwargs.get("topic")
        trefs = kwargs.get("refs")
        num = kwargs.get("num", 2)

        t = text.Term.normalize(t)

        if not trefs:
            from . import topic
            topic_manager = topic.get_topics()
            topic_obj = topic_manager.get(t)
            trefs = [pair[0] for pair in topic_obj.sources[:num]]

        normal_refs = [text.Ref(ref).normal() for ref in trefs]

        d = {"title": {"en": t, "he": hebrew_term(t)}, "refs": normal_refs}

        return d
Example #17
0
        def _create_parasha_verse_connection_story(parasha_obj,
                                                   mustHave=None,
                                                   **kwargs):
            from sefaria.utils.calendars import make_parashah_response_from_calendar_entry
            from . import ref_data

            mustHave = mustHave or []

            cal = make_parashah_response_from_calendar_entry(parasha_obj)[0]
            parasha_ref = text.Ref(parasha_obj["ref"])

            top_ref = ref_data.RefDataSet.from_ref(parasha_ref).nth_ref(
                iteration)

            connection_refs = [
                l.ref_opposite(top_ref) for l in filter(
                    lambda x: x.type != "commentary", top_ref.linkset())
            ]

            connection_ref = None
            while connection_ref is None:
                connection_ref = random.choice(connection_refs)
                category = connection_ref.index.categories[0]
                if category == "Tanakh" or category == "Reference":  # Quoting commentary isn't best for this
                    connection_ref = None
                    continue
                if not connection_ref.is_text_translated():
                    mustHave += ["readsHebrew"]

            cls.generate_story(
                refs=[top_ref.normal(),
                      connection_ref.normal()],
                title={
                    "en":
                    category + " on " + cal["displayValue"]["en"],
                    "he":
                    hebrew_term(category) + u" על " + cal["displayValue"]["he"]
                },
                lead={
                    "en": "Weekly Torah Portion",
                    "he": u'פרשת השבוע'
                },
                mustHave=mustHave,
                **kwargs).save()
Example #18
0
def construct_index():
    names = node_names()
    en_parasha = get_parsha_dict()

    root = SchemaNode()
    root.add_title('Sefat Emet', 'en', primary=True)
    root.add_title(u'שפת אמת', 'he', primary=True)
    root.key = 'Sefat Emet'

    for book in names.keys():
        book_node = SchemaNode()
        book_node.add_title(book, 'en', primary=True)
        book_node.add_title(hebrew_term(book), 'he', primary=True)
        book_node.key = book

        for parasha in names[book].keys():
            parsha_node = SchemaNode()
            parsha_node.add_title(en_parasha[parasha], 'en', primary=True)
            parsha_node.add_title(parasha, 'he', primary=True)
            parsha_node.key = en_parasha[parasha]

            for year in names[book][parasha]:
                year_node = JaggedArrayNode()
                civil_year = get_civil_year(year, book)
                year_node.add_title(civil_year, 'en', primary=True)
                year_node.add_title(fix_hebrew_years(year), 'he', primary=True)
                year_node.key = civil_year
                year_node.depth = 1
                year_node.addressTypes = ['Integer']
                year_node.sectionNames = ['Paragraph']
                parsha_node.append(year_node)
            book_node.append(parsha_node)
        root.append(book_node)
    root.validate()

    index = {
        'title': 'Sefat Emet',
        'categories': ['Chasidut'],
        'schema': root.serialize()
    }
    return index
Example #19
0
def upload_footnote_index():
    """
    Footnotes are uploaded as a commentary2 - Schema with each book a depth 2 jaggedArray
    """

    books = library.get_indexes_in_category('Tanach')

    # create index record
    record = SchemaNode()
    record.add_title(
        'JPS 1985 Footnotes',
        'en',
        primary=True,
    )
    record.add_title(
        u'הערות שוליים תרגום 1985 של JPS',
        'he',
        primary=True,
    )
    record.key = 'JPS 1985 Footnotes'

    # add nodes
    for book in books:
        node = JaggedArrayNode()
        node.add_title(book, 'en', primary=True)
        node.add_title(hebrew_term(book), 'he', primary=True)
        node.key = book
        node.depth = 2
        node.addressTypes = ['Integer', 'Integer']
        node.sectionNames = ['Chapter', 'Footnote']
        record.append(node)
    record.validate()

    index = {
        "title": "JPS 1985 Footnotes",
        "categories": ["Commentary2", "Tanach", "JPS"],
        "schema": record.serialize()
    }
    functions.post_index(index)
Example #20
0
def build_index(section_names):
    record = SchemaNode()
    record.add_title(u'Noda BeYehuda', 'en', True)
    record.add_title(u'נודע ביהודה', 'he', True)
    record.key = u'Noda BeYehuda'

    for section in section_names:
        node = JaggedArrayNode()
        node.add_title(section, 'en', True)
        node.add_title(hebrew_term(section), 'he', True)
        node.key = section
        node.depth = 2
        node.addressTypes = ['Integer', 'Integer']
        node.sectionNames = ['Teshuva', 'Paragraph']
        record.append(node)

    record.validate()

    index = {'title': 'Noda BeYehuda',
             'categories': ['Responsa'],
             'schema': record.serialize()}

    return index
Example #21
0
def construct_index():
    names = node_names()
    en_parasha = get_parsha_dict()

    root = SchemaNode()
    root.add_title('Sefat Emet', 'en', primary=True)
    root.add_title(u'שפת אמת', 'he', primary=True)
    root.key = 'Sefat Emet'

    for book in names.keys():
        book_node = SchemaNode()
        book_node.add_title(book, 'en', primary=True)
        book_node.add_title(hebrew_term(book), 'he', primary=True)
        book_node.key = book

        for parasha in names[book].keys():
            parsha_node = JaggedArrayNode()
            p_names = [
                p.contents()['name'] for p in TermSet({'scheme': "Parasha"})
            ]
            if en_parasha[parasha] in p_names:
                parsha_node.add_shared_term(en_parasha[parasha])
                parsha_node.key = en_parasha[parasha]
            else:
                parsha_node.add_primary_titles(en_parasha[parasha], parasha)
            parsha_node.add_structure(['Section', 'Comment'])
            book_node.append(parsha_node)
        root.append(book_node)
    root.validate()

    index = {
        'title': 'Sefat Emet',
        'categories': ['Chasidut'],
        'schema': root.serialize()
    }
    return index
Example #22
0
def format_link_object_for_client(link, with_text, ref, pos=None):
    """
    :param link: Link object
    :param ref: Ref object of the source of the link
    :param pos: Position of the Ref in the Link.  If not passed, it will be derived from the first two arguments.
    :return: Dict
    """
    com = {}

    # The text we're asked to get links to
    anchorTref = link.refs[pos]
    anchorRef  = Ref(anchorTref)
    anchorTrefExpanded = getattr(link, "expandedRefs{}".format(pos))

    # The link we found to anchorRef
    linkPos   = (pos + 1) % 2
    linkTref  = link.refs[linkPos]
    linkRef   = Ref(linkTref)
    langs     = getattr(link, "availableLangs", [[],[]])
    linkLangs = langs[linkPos]

    com["_id"]               = str(link._id)
    com['index_title']       = linkRef.index.title
    com["category"]          = linkRef.primary_category #usually the index's categories[0] or "Commentary".
    com["type"]              = link.type
    com["ref"]               = linkTref
    com["anchorRef"]         = anchorTref
    com["anchorRefExpanded"] = anchorTrefExpanded
    com["sourceRef"]         = linkTref
    com["sourceHeRef"]       = linkRef.he_normal()
    com["anchorVerse"]       = anchorRef.sections[-1] if len(anchorRef.sections) else 0
    com["sourceHasEn"]       = "en" in linkLangs
    # com["anchorText"]        = getattr(link, "anchorText", "") # not currently used
    if getattr(link, "inline_reference", None):
        com["inline_reference"]  = getattr(link, "inline_reference", None)
    if getattr(link, "highlightedWords", None):
        com["highlightedWords"] = getattr(link, "highlightedWords", None)

    compDate = getattr(linkRef.index, "compDate", None)
    if compDate:
        com["compDate"] = int(compDate)
        try:
            com["errorMargin"] = int(getattr(linkRef.index, "errorMargin", 0))
        except ValueError:
            com["errorMargin"] = 0

    # Pad out the sections list, so that comparison between comment numbers are apples-to-apples
    lsections = linkRef.sections[:] + [0] * (linkRef.index_node.depth - len(linkRef.sections))
    # Build a decimal comment number based on the last two digits of the section array
    com["commentaryNum"] = lsections[-1] if len(lsections) == 1 \
            else float('{0}.{1:04d}'.format(*lsections[-2:])) if len(lsections) > 1 else 0

    if with_text:
        text             = TextFamily(linkRef, context=0, commentary=False)
        com["text"]      = text.text if isinstance(text.text, basestring) else JaggedTextArray(text.text).flatten_to_array()
        com["he"]        = text.he if isinstance(text.he, basestring) else JaggedTextArray(text.he).flatten_to_array()

    # if the the link is commentary, strip redundant info (e.g. "Rashi on Genesis 4:2" -> "Rashi")
    # this is now simpler, and there is explicit data on the index record for it.
    if com["type"] == "commentary":
        com["collectiveTitle"] = {
            'en': getattr(linkRef.index, 'collective_title', linkRef.index.title),
            'he': hebrew_term(getattr(linkRef.index, 'collective_title', linkRef.index.get_title("he")))
        }
    else:
        com["collectiveTitle"] = {'en': linkRef.index.title, 'he': linkRef.index.get_title("he")}

    if com["type"] != "commentary" and com["category"] == "Commentary":
            com["category"] = "Quoting Commentary"

    if com["category"] == "Modern Works" and getattr(linkRef.index, "dependence", None) == "Commentary":
        # print "Transforming " + linkRef.normal()
        com["category"] = "Modern Commentary"
        com["collectiveTitle"] = {
            'en': getattr(linkRef.index, 'collective_title', linkRef.index.title),
            'he': hebrew_term(getattr(linkRef.index, 'collective_title', linkRef.index.get_title("he")))
        }

    if linkRef.index_node.primary_title("he"):
        com["heTitle"] = linkRef.index_node.primary_title("he")

    return com