def make_data_from_sheets(self):
if self.sources_dict and self.related_topics_dict:
# If count data was already passed down, use it
sources_dict = self.sources_dict
related_topics_dict = self.related_topics_dict
else:
# Otherwise, grab all relavant sheets and make a count
projection = {"tags": 1, "sources.ref": 1}
sheets = db.sheets.find({
"tags": self.topic,
"status": "public"
}, projection)
sources_dict = defaultdict(int)
related_topics_dict = defaultdict(int)
for sheet in sheets:
for source in sheet.get("sources", []):
if "ref" in source:
sources_dict[source["ref"]] += 1
sheet_tags = list(
set([Term.normalize(tag)
for tag in sheet.get("tags", [])]))
for tag in sheet_tags:
if tag != self.topic:
related_topics_dict[tag] += 1
self.sources = sorted(sources_dict.iteritems(),
key=lambda (k, v): v,
reverse=True)
self.related_topics = sorted(related_topics_dict.iteritems(),
key=lambda (k, v): v,
reverse=True)
def make_data_from_sheets(self):
"""
Processes all public source sheets to create topic data.
"""
tags = {}
results = []
projection = {"tags": 1, "sources.ref": 1}
sheet_list = db.sheets.find({"status": "public"}, projection)
for sheet in sheet_list:
sheet_tags = sheet.get("tags", [])
sheet_tags = list(set([Term.normalize(tag) for tag in sheet_tags]))
for tag in sheet_tags:
if tag not in tags:
tags[tag] = {
"tag": tag,
"sources_dict": defaultdict(int),
"related_topics_dict": defaultdict(int)
}
for source in sheet.get("sources", []):
if "ref" in source:
tags[tag]["sources_dict"][source["ref"]] += 1
for related_tag in sheet_tags:
if tag != related_tag:
tags[tag]["related_topics_dict"][related_tag] += 1
for tag in tags:
topic = Topic(tag, sources_dict=tags[tag]["sources_dict"], related_topics_dict=tags[tag]["related_topics_dict"])
topic.filter_sources()
if len(topic.sources) > 0:
self.topics[tag] = topic
self._loaded = True
def make_data_from_sheets(self):
"""
Processes all public source sheets to create topic data.
"""
tags = {}
results = []
projection = {"tags": 1, "sources.ref": 1}
sheet_list = db.sheets.find({"status": "public"}, projection)
for sheet in sheet_list:
sheet_tags = sheet.get("tags", [])
sheet_tags = list(set([Term.normalize(tag) for tag in sheet_tags]))
for tag in sheet_tags:
if tag not in tags:
tags[tag] = {
"tag": tag,
"sources_dict": defaultdict(int),
"related_topics_dict": defaultdict(int)
}
for source in sheet.get("sources", []):
if "ref" in source:
tags[tag]["sources_dict"][source["ref"]] += 1
for related_tag in sheet_tags:
if tag != related_tag:
tags[tag]["related_topics_dict"][related_tag] += 1
for tag in tags:
topic = Topic(tag,
sources_dict=tags[tag]["sources_dict"],
related_topics_dict=tags[tag]["related_topics_dict"])
topic.filter_sources()
if len(topic.sources) > 0:
self.topics[tag] = topic
self._loaded = True
def export_hebrew_categories():
"""
Writes translation of all English categories into a single file.
"""
print "Export Hebrew Categories"
term = Term()
eng_cats = model.library.get_text_categories()
hebrew_cats_json = {}
for e in eng_cats:
t = term.load_by_title(e)
if not t:
print u"Couldn't load term '{}'. Skipping Hebrew category".format(
e)
else:
hebrew_cats_json[e] = t.titles[1][u'text']
write_doc(hebrew_cats_json, HEB_CATS_PATH)
def get_refs(self, node):
assert isinstance(node, SchemaNode)
if node.is_leaf():
nodes = [node]
else:
nodes = node.get_leaf_nodes()
for leaf_node in nodes:
assert isinstance(leaf_node, JaggedArrayNode)
if leaf_node.sharedTitle is not None:
term = Term().load({'name': leaf_node.sharedTitle})
self.base_refs.append(Ref(term.ref))
self.mei_refs.append(leaf_node.ref())
else:
for subref in leaf_node.ref().all_subrefs():
assert isinstance(subref, Ref)
if not subref.is_section_level(
): # Don't bother trying to match depth 1 texts
break
if subref.is_empty():
continue
else:
base_book = leaf_node.primary_title('en')
base_chapter = subref.sections[0]
self.base_refs.append(
Ref("{} {}".format(base_book, base_chapter)))
self.mei_refs.append(subref)
def export_hebrew_categories(for_sources=False):
"""
Writes translation of all English categories into a single file.
"""
print("Export Hebrew Categories")
term = Term()
eng_cats = model.library.get_text_categories()
hebrew_cats_json = {}
for e in eng_cats:
t = term.load_by_title(e)
if not t:
print("Couldn't load term '{}'. Skipping Hebrew category".format(e))
else:
hebrew_cats_json[e] = t.titles[1]['text']
write_doc(hebrew_cats_json, (SEFARIA_IOS_SOURCES_PATH if for_sources else EXPORT_PATH) + HEB_CATS_PATH)
write_doc(hebrew_cats_json, (SEFARIA_ANDROID_SOURCES_PATH if for_sources else EXPORT_PATH) + HEB_CATS_PATH)
def get_hok_parasha(datetime_obj, diaspora=diaspora):
parasha = get_parasha(datetime_obj, diaspora=diaspora)['parasha']
parasha = parasha.replace('Lech-Lecha', 'Lech Lecha')
parasha = parasha.split('-')[0]
if parasha == 'Shmini Atzeret':
parasha = "V'Zot HaBerachah"
parasha_term = Term().load({
'category': 'Torah Portions',
'titles': {
'$elemMatch': {
'text': parasha
}
}
})
if not parasha_term:
parasha_term = get_hok_parasha(datetime_obj +
datetime.timedelta(7),
diaspora=diaspora)
return parasha_term
def make_data_from_sheets(self):
if self.sources_dict and self.related_topics_dict:
# If count data was already passed down, use it
sources_dict = self.sources_dict
related_topics_dict = self.related_topics_dict
else:
# Otherwise, grab all relavant sheets and make a count
projection = {"tags": 1, "sources.ref": 1}
sheets = db.sheets.find({"tags": self.topic, "status": "public"}, projection)
sources_dict = defaultdict(int)
related_topics_dict = defaultdict(int)
for sheet in sheets:
for source in sheet.get("sources", []):
if "ref" in source:
sources_dict[source["ref"]] += 1
sheet_tags = list(set([Term.normalize(tag) for tag in sheet.get("tags", [])]))
for tag in sheet_tags:
if tag != self.topic:
related_topics_dict[tag] += 1
self.sources = sorted(sources_dict.iteritems(), key=lambda (k,v): v, reverse=True)
self.related_topics = sorted(related_topics_dict.iteritems(), key=lambda (k,v): v, reverse=True)